From edadf145f3846c95b7faf48c4543bbd15765231f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 20 Sep 2014 16:16:13 -0700 Subject: [PATCH 001/365] Initial Cookiecutter Commit. --- .coveragerc | 7 + .gitignore | 52 ++++++ .gitreview | 4 + .mailmap | 3 + .testr.conf | 7 + CONTRIBUTING.rst | 17 ++ HACKING.rst | 4 + LICENSE | 175 ++++++++++++++++++ MANIFEST.in | 6 + README.rst | 15 ++ babel.cfg | 1 + doc/source/conf.py | 75 ++++++++ doc/source/contributing.rst | 4 + doc/source/index.rst | 24 +++ doc/source/installation.rst | 12 ++ doc/source/readme.rst | 1 + doc/source/usage.rst | 7 + openstack-common.conf | 6 + os_client_config/__init__.py | 19 ++ os_client_config/tests/__init__.py | 0 os_client_config/tests/base.py | 23 +++ .../tests/test_os_client_config.py | 28 +++ requirements.txt | 6 + setup.cfg | 47 +++++ setup.py | 22 +++ test-requirements.txt | 15 ++ tox.ini | 34 ++++ 27 files changed, 614 insertions(+) create mode 100644 .coveragerc create mode 100644 .gitignore create mode 100644 .gitreview create mode 100644 .mailmap create mode 100644 .testr.conf create mode 100644 CONTRIBUTING.rst create mode 100644 HACKING.rst create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 babel.cfg create mode 100755 doc/source/conf.py create mode 100644 doc/source/contributing.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/installation.rst create mode 100644 doc/source/readme.rst create mode 100644 doc/source/usage.rst create mode 100644 openstack-common.conf create mode 100644 os_client_config/__init__.py create mode 100644 os_client_config/tests/__init__.py create mode 100644 os_client_config/tests/base.py create mode 100644 os_client_config/tests/test_os_client_config.py create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100755 setup.py create mode 100644 test-requirements.txt create mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..5f0c7fd8b --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +branch = True +source = os_client_config +omit = os_client_config/tests/*,os_client_config/openstack/* + +[report] +ignore-errors = True \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..e6a97ec61 --- /dev/null +++ b/.gitignore @@ -0,0 +1,52 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? \ No newline at end of file diff --git a/.gitreview b/.gitreview new file mode 100644 index 000000000..69bff3ae3 --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/os-client-config.git \ No newline at end of file diff --git a/.mailmap b/.mailmap new file mode 100644 index 000000000..cc92f17b8 --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# +# \ No newline at end of file diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 000000000..fb622677a --- /dev/null +++ b/.testr.conf @@ -0,0 +1,7 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list \ No newline at end of file diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 000000000..c1f3dc297 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,17 @@ +If you would like to contribute to the development of OpenStack, +you must follow the steps in the "If you're a developer, start here" +section of this page: + + http://wiki.openstack.org/HowToContribute + +Once those steps have been completed, changes to OpenStack +should be submitted for review via the Gerrit tool, following +the workflow documented at: + + http://wiki.openstack.org/GerritWorkflow + +Pull requests submitted through GitHub will be ignored. + +Bugs should be filed on Launchpad, not GitHub: + + https://bugs.launchpad.net/os-client-config \ No newline at end of file diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 000000000..c995c5c92 --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,4 @@ +os-client-config Style Commandments +=============================================== + +Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..67db85882 --- /dev/null +++ b/LICENSE @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..90f8a7aef --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include AUTHORS +include ChangeLog +exclude .gitignore +exclude .gitreview + +global-exclude *.pyc \ No newline at end of file diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..cf2315d32 --- /dev/null +++ b/README.rst @@ -0,0 +1,15 @@ +=============================== +os-client-config +=============================== + +OpenStack Client Configuation Library + +* Free software: Apache license +* Documentation: http://docs.openstack.org/developer/os-client-config +* Source: http://git.openstack.org/cgit/openstack/os-client-config +* Bugs: http://bugs.launchpad.net/os-client-config + +Features +-------- + +* TODO \ No newline at end of file diff --git a/babel.cfg b/babel.cfg new file mode 100644 index 000000000..efceab818 --- /dev/null +++ b/babel.cfg @@ -0,0 +1 @@ +[python: **.py] diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100755 index 000000000..02be16776 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + #'sphinx.ext.intersphinx', + 'oslosphinx' +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'os-client-config' +copyright = u'2013, OpenStack Foundation' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +# html_static_path = ['static'] + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + u'%s Documentation' % project, + u'OpenStack Foundation', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} \ No newline at end of file diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst new file mode 100644 index 000000000..ed77c1262 --- /dev/null +++ b/doc/source/contributing.rst @@ -0,0 +1,4 @@ +============ +Contributing +============ +.. include:: ../../CONTRIBUTING.rst \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..2c8b52d5e --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,24 @@ +.. os-client-config documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to os-client-config's documentation! +======================================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + installation + usage + contributing + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/source/installation.rst b/doc/source/installation.rst new file mode 100644 index 000000000..48bbc2f20 --- /dev/null +++ b/doc/source/installation.rst @@ -0,0 +1,12 @@ +============ +Installation +============ + +At the command line:: + + $ pip install os-client-config + +Or, if you have virtualenvwrapper installed:: + + $ mkvirtualenv os-client-config + $ pip install os-client-config \ No newline at end of file diff --git a/doc/source/readme.rst b/doc/source/readme.rst new file mode 100644 index 000000000..38ba8043d --- /dev/null +++ b/doc/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.rst \ No newline at end of file diff --git a/doc/source/usage.rst b/doc/source/usage.rst new file mode 100644 index 000000000..910fd0790 --- /dev/null +++ b/doc/source/usage.rst @@ -0,0 +1,7 @@ +======== +Usage +======== + +To use os-client-config in a project:: + + import os_client_config \ No newline at end of file diff --git a/openstack-common.conf b/openstack-common.conf new file mode 100644 index 000000000..e8eb2aa2c --- /dev/null +++ b/openstack-common.conf @@ -0,0 +1,6 @@ +[DEFAULT] + +# The list of modules to copy from oslo-incubator.git + +# The base module to hold the copy of openstack.common +base=os_client_config \ No newline at end of file diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py new file mode 100644 index 000000000..26bdf7e34 --- /dev/null +++ b/os_client_config/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pbr.version + + +__version__ = pbr.version.VersionInfo( + 'os_client_config').version_string() \ No newline at end of file diff --git a/os_client_config/tests/__init__.py b/os_client_config/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py new file mode 100644 index 000000000..185fd6fd9 --- /dev/null +++ b/os_client_config/tests/base.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright 2010-2011 OpenStack Foundation +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslotest import base + + +class TestCase(base.BaseTestCase): + + """Test case base class for all unit tests.""" \ No newline at end of file diff --git a/os_client_config/tests/test_os_client_config.py b/os_client_config/tests/test_os_client_config.py new file mode 100644 index 000000000..9b26ad547 --- /dev/null +++ b/os_client_config/tests/test_os_client_config.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +test_os_client_config +---------------------------------- + +Tests for `os_client_config` module. +""" + +from os_client_config.tests import base + + +class TestOs_client_config(base.TestCase): + + def test_something(self): + pass \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..bc7131e80 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +pbr>=0.6,!=0.7,<1.0 +Babel>=1.3 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..269052910 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,47 @@ +[metadata] +name = os-client-config +summary = OpenStack Client Configuation Library +description-file = + README.rst +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = http://www.openstack.org/ +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.4 + +[files] +packages = + os_client_config + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 + +[upload_sphinx] +upload-dir = doc/build/html + +[compile_catalog] +directory = os_client_config/locale +domain = os-client-config + +[update_catalog] +domain = os-client-config +output_dir = os_client_config/locale +input_file = os_client_config/locale/os-client-config.pot + +[extract_messages] +keywords = _ gettext ngettext l_ lazy_gettext +mapping_file = babel.cfg +output_file = os_client_config/locale/os-client-config.pot \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..7eeb36b53 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 000000000..7b79352b4 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,15 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +hacking>=0.9.2,<0.10 + +coverage>=3.6 +discover +python-subunit +sphinx>=1.1.2 +oslosphinx +oslotest>=1.1.0.0a1 +testrepository>=0.0.18 +testscenarios>=0.4 +testtools>=0.9.34 \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..9be310a4c --- /dev/null +++ b/tox.ini @@ -0,0 +1,34 @@ +[tox] +minversion = 1.6 +envlist = py33,py34,py26,py27,pypy,pep8 +skipsdist = True + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = python setup.py testr --slowest --testr-args='{posargs}' + +[testenv:pep8] +commands = flake8 + +[testenv:venv] +commands = {posargs} + +[testenv:cover] +commands = python setup.py testr --coverage --testr-args='{posargs}' + +[testenv:docs] +commands = python setup.py build_sphinx + +[flake8] +# H803 skipped on purpose per list discussion. +# E123, E125 skipped as they are invalid PEP-8. + +show-source = True +ignore = E123,E125,H803 +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build \ No newline at end of file From 6efe00dbf3959ebfbb49045f357823da0b94c3e0 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 21 Sep 2014 12:16:20 -0700 Subject: [PATCH 002/365] Port in config reading from shade --- README.rst | 109 ++++++++++++++++++++-- os_client_config/__init__.py | 10 +- os_client_config/cloud_config.py | 20 ++++ os_client_config/config.py | 150 ++++++++++++++++++++++++++++++ os_client_config/defaults_dict.py | 27 ++++++ os_client_config/exceptions.py | 17 ++++ os_client_config/vendors.py | 25 +++++ setup.py | 2 +- 8 files changed, 346 insertions(+), 14 deletions(-) create mode 100644 os_client_config/cloud_config.py create mode 100644 os_client_config/config.py create mode 100644 os_client_config/defaults_dict.py create mode 100644 os_client_config/exceptions.py create mode 100644 os_client_config/vendors.py diff --git a/README.rst b/README.rst index cf2315d32..6332f8391 100644 --- a/README.rst +++ b/README.rst @@ -2,14 +2,111 @@ os-client-config =============================== -OpenStack Client Configuation Library +os-client-config is a library for collecting client configuration for +using an OpenStack cloud in a consistent and comprehensive manner. It +will find cloud config for as few as 1 cloud and as many as you want to +put in a config file. It will read environment variables and config files, +and it also contains some vendor specific default values so that you don't +have to know extra info to use OpenStack + +Environment Variables +--------------------- + +os-client-config honors all of the normal `OS_*` variables. It does not +provide backwards compatibility to service-specific variables such as +`NOVA_USERNAME`. + +If you have environment variables and no config files, os-client-config +will produce a cloud config object named "openstack" containing your +values from the environment. + +Service specific settings, like the nova service type, are set with the +default service type as a prefix. For instance, to set a special service_type +for trove (because you're using Rackspace) set: +:: + + export OS_DATABASE_SERVICE_TYPE=rax:database + +Config Files +------------ + +os-client-config will for a file called clouds.yaml in the following locations: +* Current Directory +* ~/.config/openstack +* /etc/openstack + +The keys are all of the keys you'd expect from `OS_*` - except lower case +and without the OS prefix. So, username is set with `username`. + +Service specific settings, like the nova service type, are set with the +default service type as a prefix. For instance, to set a special service_type +for trove (because you're using Rackspace) set: +:: + + database_service_type: 'rax:database' + +An example config file is probably helpful: +:: + + clouds: + mordred: + cloud: hp + username: mordred@inaugust.com + password: XXXXXXXXX + project_id: mordred@inaugust.com + region_name: region-b.geo-1 + monty: + auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 + username: monty.taylor@hp.com + password: XXXXXXXX + project_id: monty.taylor@hp.com-default-tenant + region_name: region-b.geo-1 + infra: + cloud: rackspace + username: openstackci + password: XXXXXXXX + project_id: 610275 + region_name: DFW,ORD,IAD + +You may note a few things. First, since auth_url settings are silly +and embarrasingly ugly, known cloud vendors are included and may be referrenced +by name. One of the benefits of that is that auth_url isn't the only thing +the vendor defaults contain. For instance, since Rackspace is broken and lists +`rax:database` as the service type for trove, os-client-config knows that +so that you don't have to. + +Also, region_name can be a list of regions. When you call get_all_clouds, +you'll get a cloud config object for each cloud/region combo. + +Usage +----- + +The simplest and least useful thing you can do is: +:: + + python -m os_client_config.config + +Which will print out whatever if finds for your config. If you want to use +it from python, which is much more likely what you want to do, things like: + +Get a named cloud. +:: + + import os_client_config + + cloud_config = os_client_config.OpenStackConfig().get_one_cloud( + 'hp', 'region-b.geo-1') + print(cloud_config.name, cloud_config.region, cloud_config.config) + +Or, get all of the clouds. +:: + import os_client_config + + cloud_config = os_client_config.OpenStackConfig().get_all_clouds() + for cloud in cloud_config: + print(cloud.name, cloud.region, cloud.config) * Free software: Apache license * Documentation: http://docs.openstack.org/developer/os-client-config * Source: http://git.openstack.org/cgit/openstack/os-client-config * Bugs: http://bugs.launchpad.net/os-client-config - -Features --------- - -* TODO \ No newline at end of file diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index 26bdf7e34..d5fd36cb6 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- - +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,8 +12,4 @@ # License for the specific language governing permissions and limitations # under the License. -import pbr.version - - -__version__ = pbr.version.VersionInfo( - 'os_client_config').version_string() \ No newline at end of file +from os_client_config.config import OpenStackConfig # noqa diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py new file mode 100644 index 000000000..d0c932f41 --- /dev/null +++ b/os_client_config/cloud_config.py @@ -0,0 +1,20 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class CloudConfig(object): + def __init__(self, name, region, config): + self.name = name + self.region = region + self.config = config diff --git a/os_client_config/config.py b/os_client_config/config.py new file mode 100644 index 000000000..1742b5b16 --- /dev/null +++ b/os_client_config/config.py @@ -0,0 +1,150 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import os + +import yaml + +from os_client_config import cloud_config +from os_client_config import defaults_dict +from os_client_config import exceptions +from os_client_config import vendors + +CONFIG_HOME = os.path.join(os.path.expanduser( + os.environ.get('XDG_CONFIG_HOME', os.path.join('~', '.config'))), + 'openstack') +CONFIG_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] +CONFIG_FILES = [ + os.path.join(d, 'clouds.yaml') for d in CONFIG_SEARCH_PATH] +BOOL_KEYS = ('insecure', 'cache') +REQUIRED_VALUES = ('auth_url', 'username', 'password', 'project_id') +SERVICES = ( + 'compute', 'identity', 'network', 'metering', 'object-store', + 'volume', 'dns', 'image', 'database') + + +def get_boolean(value): + if value.lower() == 'true': + return True + return False + + +class OpenStackConfig(object): + + def __init__(self, config_files=None): + self._config_files = config_files or CONFIG_FILES + + defaults = defaults_dict.DefaultsDict() + defaults.add('username') + defaults.add('user_domain_name') + defaults.add('password') + defaults.add('project_id', defaults['username'], also='tenant_name') + defaults.add('project_domain_name') + defaults.add('auth_url') + defaults.add('region_name') + defaults.add('cache', 'false') + defaults.add('auth_token') + defaults.add('insecure', 'false') + defaults.add('cacert') + + for service in SERVICES: + defaults.add('service_name', prefix=service) + defaults.add('service_type', prefix=service) + defaults.add('endpoint_type', prefix=service) + defaults.add('endpoint', prefix=service) + self.defaults = defaults + + # use a config file if it exists where expected + self.cloud_config = self._load_config_file() + if not self.cloud_config: + self.cloud_config = dict( + clouds=dict(openstack=dict(self.defaults))) + + @classmethod + def get_services(klass): + return SERVICES + + def _load_config_file(self): + for path in self._config_files: + if os.path.exists(path): + return yaml.load(open(path, 'r')) + + def _get_regions(self, cloud): + return self.cloud_config['clouds'][cloud]['region_name'] + + def _get_region(self, cloud): + return self._get_regions(cloud).split(',')[0] + + def _get_cloud_sections(self): + return self.cloud_config['clouds'].keys() + + def _get_base_cloud_config(self, name): + cloud = dict() + if name in self.cloud_config['clouds']: + our_cloud = self.cloud_config['clouds'][name] + else: + our_cloud = dict() + + # yes, I know the next line looks silly + if 'cloud' in our_cloud: + cloud.update(vendors.CLOUD_DEFAULTS[our_cloud['cloud']]) + + cloud.update(self.defaults) + cloud.update(our_cloud) + if 'cloud' in cloud: + del cloud['cloud'] + return cloud + + def get_all_clouds(self): + + clouds = [] + + for cloud in self._get_cloud_sections(): + for region in self._get_regions(cloud).split(','): + clouds.append(self.get_one_cloud(cloud, region)) + return clouds + + def get_one_cloud(self, name='openstack', region=None): + + if not region: + region = self._get_region(name) + + config = self._get_base_cloud_config(name) + config['region_name'] = region + + for key in BOOL_KEYS: + if key in config: + config[key] = get_boolean(config[key]) + + for key in REQUIRED_VALUES: + if key not in config or not config[key]: + raise exceptions.OpenStackConfigException( + 'Unable to find full auth information for cloud {name} in' + ' config files {files}' + ' or environment variables.'.format( + name=name, files=','.join(self._config_files))) + + # If any of the defaults reference other values, we need to expand + for (key, value) in config.items(): + if hasattr(value, 'format'): + config[key] = value.format(**config) + + return cloud_config.CloudConfig( + name=name, region=region, config=config) + +if __name__ == '__main__': + config = OpenStackConfig().get_all_clouds() + for cloud in config: + print(cloud.name, cloud.region, cloud.config) diff --git a/os_client_config/defaults_dict.py b/os_client_config/defaults_dict.py new file mode 100644 index 000000000..43de77beb --- /dev/null +++ b/os_client_config/defaults_dict.py @@ -0,0 +1,27 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + + +class DefaultsDict(dict): + + def add(self, key, default_value=None, also=None, prefix=None): + if prefix: + key = '%s_%s' % (prefix.replace('-', '_'), key) + if also: + value = os.environ.get(also, default_value) + value = os.environ.get('OS_%s' % key.upper(), default_value) + if value is not None: + self.__setitem__(key, value) diff --git a/os_client_config/exceptions.py b/os_client_config/exceptions.py new file mode 100644 index 000000000..ab78dc2e5 --- /dev/null +++ b/os_client_config/exceptions.py @@ -0,0 +1,17 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class OpenStackConfigException(Exception): + """Something went wrong with parsing your OpenStack Config.""" diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py new file mode 100644 index 000000000..d1b29a588 --- /dev/null +++ b/os_client_config/vendors.py @@ -0,0 +1,25 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +CLOUD_DEFAULTS = dict( + hp=dict( + auth_url='https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0', + region_name='region-b.geo-1', + ), + rackspace=dict( + auth_url='https://identity.api.rackspacecloud.com/v2.0/', + image_endpoint='https://{region_name}.images.api.rackspacecloud.com/', + database_service_type='rax:database', + ) +) diff --git a/setup.py b/setup.py index 7eeb36b53..70c2b3f32 100755 --- a/setup.py +++ b/setup.py @@ -19,4 +19,4 @@ import setuptools setuptools.setup( setup_requires=['pbr'], - pbr=True) \ No newline at end of file + pbr=True) From 9bbb4f30f49eb157d31a9aa1c37aca3004656009 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 21 Sep 2014 12:20:55 -0700 Subject: [PATCH 003/365] Remove babel and add pyyaml --- babel.cfg | 1 - requirements.txt | 3 ++- setup.cfg | 14 -------------- test-requirements.txt | 2 +- 4 files changed, 3 insertions(+), 17 deletions(-) delete mode 100644 babel.cfg diff --git a/babel.cfg b/babel.cfg deleted file mode 100644 index efceab818..000000000 --- a/babel.cfg +++ /dev/null @@ -1 +0,0 @@ -[python: **.py] diff --git a/requirements.txt b/requirements.txt index bc7131e80..9cf3f8ae0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ # process, which may cause wedges in the gate later. pbr>=0.6,!=0.7,<1.0 -Babel>=1.3 \ No newline at end of file + +PyYAML diff --git a/setup.cfg b/setup.cfg index 269052910..df3434f0f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,17 +31,3 @@ all_files = 1 [upload_sphinx] upload-dir = doc/build/html - -[compile_catalog] -directory = os_client_config/locale -domain = os-client-config - -[update_catalog] -domain = os-client-config -output_dir = os_client_config/locale -input_file = os_client_config/locale/os-client-config.pot - -[extract_messages] -keywords = _ gettext ngettext l_ lazy_gettext -mapping_file = babel.cfg -output_file = os_client_config/locale/os-client-config.pot \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt index 7b79352b4..d494076d9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,4 +12,4 @@ oslosphinx oslotest>=1.1.0.0a1 testrepository>=0.0.18 testscenarios>=0.4 -testtools>=0.9.34 \ No newline at end of file +testtools>=0.9.34 From 69d2a3e0ad97a20cec498ab87c81af5e11e5c79c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 21 Sep 2014 14:17:38 -0700 Subject: [PATCH 004/365] Get rid of extra complexity with service values We don't need to enumerate the service types - we can simply match at consumption time on patterns. --- os_client_config/config.py | 13 +++---------- os_client_config/vendors.py | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1742b5b16..5cc8edd78 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -30,9 +30,6 @@ CONFIG_FILES = [ os.path.join(d, 'clouds.yaml') for d in CONFIG_SEARCH_PATH] BOOL_KEYS = ('insecure', 'cache') REQUIRED_VALUES = ('auth_url', 'username', 'password', 'project_id') -SERVICES = ( - 'compute', 'identity', 'network', 'metering', 'object-store', - 'volume', 'dns', 'image', 'database') def get_boolean(value): @@ -54,16 +51,12 @@ class OpenStackConfig(object): defaults.add('project_domain_name') defaults.add('auth_url') defaults.add('region_name') - defaults.add('cache', 'false') + defaults.add('cache') defaults.add('auth_token') - defaults.add('insecure', 'false') + defaults.add('insecure') + defaults.add('endpoint_type') defaults.add('cacert') - for service in SERVICES: - defaults.add('service_name', prefix=service) - defaults.add('service_type', prefix=service) - defaults.add('endpoint_type', prefix=service) - defaults.add('endpoint', prefix=service) self.defaults = defaults # use a config file if it exists where expected diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index d1b29a588..71cf15896 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -16,10 +16,10 @@ CLOUD_DEFAULTS = dict( hp=dict( auth_url='https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0', region_name='region-b.geo-1', + dns_service_type='hp:dns', ), rackspace=dict( auth_url='https://identity.api.rackspacecloud.com/v2.0/', - image_endpoint='https://{region_name}.images.api.rackspacecloud.com/', database_service_type='rax:database', ) ) From 1277d4cfc58a0c4e7338e29ada0df9631d62da21 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 21 Sep 2014 14:33:07 -0700 Subject: [PATCH 005/365] Update the README file for more completeness --- README.rst | 16 ++++++++++++++-- os_client_config/vendors.py | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 6332f8391..33327e20b 100644 --- a/README.rst +++ b/README.rst @@ -30,11 +30,15 @@ for trove (because you're using Rackspace) set: Config Files ------------ -os-client-config will for a file called clouds.yaml in the following locations: +os-client-config will look for a file called clouds.yaml in the following +locations: + * Current Directory * ~/.config/openstack * /etc/openstack +The first file found wins. + The keys are all of the keys you'd expect from `OS_*` - except lower case and without the OS prefix. So, username is set with `username`. @@ -55,12 +59,14 @@ An example config file is probably helpful: password: XXXXXXXXX project_id: mordred@inaugust.com region_name: region-b.geo-1 + dns_service_type: hpext:dns monty: auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 username: monty.taylor@hp.com password: XXXXXXXX project_id: monty.taylor@hp.com-default-tenant region_name: region-b.geo-1 + dns_service_type: hpext:dns infra: cloud: rackspace username: openstackci @@ -71,13 +77,19 @@ An example config file is probably helpful: You may note a few things. First, since auth_url settings are silly and embarrasingly ugly, known cloud vendors are included and may be referrenced by name. One of the benefits of that is that auth_url isn't the only thing -the vendor defaults contain. For instance, since Rackspace is broken and lists +the vendor defaults contain. For instance, since Rackspace lists `rax:database` as the service type for trove, os-client-config knows that so that you don't have to. Also, region_name can be a list of regions. When you call get_all_clouds, you'll get a cloud config object for each cloud/region combo. +As seen with `dns_service_type`, any setting that makes sense to be per-service, +like `service_type` or `endpoint` or `api_version` can be set by prefixing +the setting with the default service type. That might strike you funny when +setting `service_type` and it does me too - but that's just the world we live +in. + Usage ----- diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index 71cf15896..c78aaca54 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -16,10 +16,11 @@ CLOUD_DEFAULTS = dict( hp=dict( auth_url='https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0', region_name='region-b.geo-1', - dns_service_type='hp:dns', + dns_service_type='hpext:dns', ), rackspace=dict( auth_url='https://identity.api.rackspacecloud.com/v2.0/', database_service_type='rax:database', + image_api_version='2', ) ) From 1113d0523d1f35af11ac17112a6ac7cd2a1a085a Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Sun, 21 Sep 2014 18:28:15 -0500 Subject: [PATCH 006/365] Handle null region Not all clouds define/require region_name to be set --- os_client_config/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 5cc8edd78..208b18a79 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -56,6 +56,7 @@ class OpenStackConfig(object): defaults.add('insecure') defaults.add('endpoint_type') defaults.add('cacert') + defaults.add('auth_type') self.defaults = defaults @@ -75,7 +76,11 @@ class OpenStackConfig(object): return yaml.load(open(path, 'r')) def _get_regions(self, cloud): - return self.cloud_config['clouds'][cloud]['region_name'] + try: + return self.cloud_config['clouds'][cloud]['region_name'] + except KeyError: + # No region configured + return '' def _get_region(self, cloud): return self._get_regions(cloud).split(',')[0] From b8382c349687b09e15f92cde9d0099fea5252657 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Sun, 21 Sep 2014 20:44:44 -0500 Subject: [PATCH 007/365] Make env vars lowest priority When working with multiple tools the project CLIs only know about options and environment variables. When selecting a cloud config that includes a section from vendors.py environment vars overwrite that data if they are defined. The priority order should be: * command line args * cloud config selection * environment variables --- os_client_config/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 5cc8edd78..64eaff789 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -90,11 +90,13 @@ class OpenStackConfig(object): else: our_cloud = dict() + # Get the defaults (including env vars) first + cloud.update(self.defaults) + # yes, I know the next line looks silly if 'cloud' in our_cloud: cloud.update(vendors.CLOUD_DEFAULTS[our_cloud['cloud']]) - cloud.update(self.defaults) cloud.update(our_cloud) if 'cloud' in cloud: del cloud['cloud'] From 7bd5ff625f9c845e1199878caf15ed128a31bb52 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Sun, 21 Sep 2014 22:16:02 -0500 Subject: [PATCH 008/365] Handle missing vendor key Continue on if the configured vendor config is not present --- os_client_config/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 5cc8edd78..5b8c53aa0 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -92,7 +92,11 @@ class OpenStackConfig(object): # yes, I know the next line looks silly if 'cloud' in our_cloud: - cloud.update(vendors.CLOUD_DEFAULTS[our_cloud['cloud']]) + try: + cloud.update(vendors.CLOUD_DEFAULTS[our_cloud['cloud']]) + except KeyError: + # Can't find the requested vendor config, go about business + pass cloud.update(self.defaults) cloud.update(our_cloud) From cb9e5059f6f9557fc9d62e12f4237719fb172e64 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 22 Sep 2014 09:01:40 -0700 Subject: [PATCH 009/365] Prep for move to stackforge --- .gitreview | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitreview b/.gitreview index 69bff3ae3..234652006 100644 --- a/.gitreview +++ b/.gitreview @@ -1,4 +1,4 @@ [gerrit] host=review.openstack.org port=29418 -project=openstack/os-client-config.git \ No newline at end of file +project=stackforge/os-client-config.git From 2c2a2953771d6599cbb3b65500c1f55eb44cafe9 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Mon, 22 Sep 2014 19:54:57 -0500 Subject: [PATCH 010/365] Add clouds-public.yaml Put vendor config outside of the code in clouds-public.yaml. Fall back to vendors.py if clouds-public.yaml not found. The search follows the same rules as clouds.yaml, the file is the same format except the top-level key is 'public-clouds'. Typically only auth_url and region_name are populated. --- os_client_config/config.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 56992df04..6612bffdd 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -30,6 +30,9 @@ CONFIG_FILES = [ os.path.join(d, 'clouds.yaml') for d in CONFIG_SEARCH_PATH] BOOL_KEYS = ('insecure', 'cache') REQUIRED_VALUES = ('auth_url', 'username', 'password', 'project_id') +VENDOR_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] +VENDOR_FILES = [ + os.path.join(d, 'clouds-public.yaml') for d in VENDOR_SEARCH_PATH] def get_boolean(value): @@ -40,8 +43,9 @@ def get_boolean(value): class OpenStackConfig(object): - def __init__(self, config_files=None): + def __init__(self, config_files=None, vendor_files=None): self._config_files = config_files or CONFIG_FILES + self._vendor_files = vendor_files or VENDOR_FILES defaults = defaults_dict.DefaultsDict() defaults.add('username') @@ -75,6 +79,11 @@ class OpenStackConfig(object): if os.path.exists(path): return yaml.load(open(path, 'r')) + def _load_vendor_file(self): + for path in self._vendor_files: + if os.path.exists(path): + return yaml.load(open(path, 'r')) + def _get_regions(self, cloud): try: return self.cloud_config['clouds'][cloud]['region_name'] @@ -100,11 +109,15 @@ class OpenStackConfig(object): # yes, I know the next line looks silly if 'cloud' in our_cloud: - try: - cloud.update(vendors.CLOUD_DEFAULTS[our_cloud['cloud']]) - except KeyError: - # Can't find the requested vendor config, go about business - pass + vendor_file = self._load_vendor_file() + if our_cloud['cloud'] in vendor_file['public-clouds']: + cloud.update(vendor_file['public-clouds'][our_cloud['cloud']]) + else: + try: + cloud.update(vendors.CLOUD_DEFAULTS[our_cloud['cloud']]) + except KeyError: + # Can't find the requested vendor config, go about business + pass cloud.update(our_cloud) if 'cloud' in cloud: From 67f1fbd22ed5d0558a92c49c5114a292d1f0cc01 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 10 Oct 2014 15:24:15 -0700 Subject: [PATCH 011/365] Remove unused class method get_services Change-Id: Id133bc3c39b97a4489e75c3d38df601f999e8f3a --- os_client_config/config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 6612bffdd..8cdb06c73 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -70,10 +70,6 @@ class OpenStackConfig(object): self.cloud_config = dict( clouds=dict(openstack=dict(self.defaults))) - @classmethod - def get_services(klass): - return SERVICES - def _load_config_file(self): for path in self._config_files: if os.path.exists(path): From 215425f421cb4b831f1a294d73a25a14f301c1f4 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 28 Sep 2014 11:49:13 -0700 Subject: [PATCH 012/365] Handle no vendor clouds config files Change-Id: If0ab1db3df8ba3a2880473f2287ae3f85c84d9d5 --- os_client_config/config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 8cdb06c73..ade970891 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -105,12 +105,13 @@ class OpenStackConfig(object): # yes, I know the next line looks silly if 'cloud' in our_cloud: + cloud_name = our_cloud['cloud'] vendor_file = self._load_vendor_file() - if our_cloud['cloud'] in vendor_file['public-clouds']: - cloud.update(vendor_file['public-clouds'][our_cloud['cloud']]) + if vendor_file and cloud_name in vendor_file['public-clouds']: + cloud.update(vendor_file['public-clouds'][cloud_name]) else: try: - cloud.update(vendors.CLOUD_DEFAULTS[our_cloud['cloud']]) + cloud.update(vendors.CLOUD_DEFAULTS[cloud_name]) except KeyError: # Can't find the requested vendor config, go about business pass From b1bb75a69b491b26048fd18c40a7ac87043ea93c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 28 Sep 2014 11:18:31 -0700 Subject: [PATCH 013/365] Add cache control settings Things need to do local caching, which means they need to control some settings about that. Add simple cache settings support. Change-Id: I7b56cc25ebe7a803816d95b79d0329f8e42025ba --- README.rst | 24 ++++++++++++++++++++++++ os_client_config/config.py | 17 +++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/README.rst b/README.rst index 33327e20b..30819f398 100644 --- a/README.rst +++ b/README.rst @@ -45,11 +45,13 @@ and without the OS prefix. So, username is set with `username`. Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type for trove (because you're using Rackspace) set: + :: database_service_type: 'rax:database' An example config file is probably helpful: + :: clouds: @@ -90,6 +92,28 @@ the setting with the default service type. That might strike you funny when setting `service_type` and it does me too - but that's just the world we live in. +Cache Settings +-------------- + +Accessing a cloud is often expensive, so it's quite common to want to do some +client-side caching of those operations. To facilitate that, os-client-config +understands a simple set of cache control settings. + +:: + + cache: + path: ~/.cache/openstack + max_age: 300 + clouds: + mordred: + cloud: hp + username: mordred@inaugust.com + password: XXXXXXXXX + project_id: mordred@inaugust.com + region_name: region-b.geo-1 + dns_service_type: hpext:dns + + Usage ----- diff --git a/os_client_config/config.py b/os_client_config/config.py index ade970891..f92c7909b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -28,6 +28,9 @@ CONFIG_HOME = os.path.join(os.path.expanduser( CONFIG_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] CONFIG_FILES = [ os.path.join(d, 'clouds.yaml') for d in CONFIG_SEARCH_PATH] +CACHE_PATH = os.path.join(os.path.expanduser( + os.environ.get('XDG_CACHE_PATH', os.path.join('~', '.cache'))), + 'openstack') BOOL_KEYS = ('insecure', 'cache') REQUIRED_VALUES = ('auth_url', 'username', 'password', 'project_id') VENDOR_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] @@ -70,6 +73,14 @@ class OpenStackConfig(object): self.cloud_config = dict( clouds=dict(openstack=dict(self.defaults))) + self._cache_max_age = 300 + self._cache_path = CACHE_PATH + if 'cache' in self.cloud_config: + self._cache_max_age = self.cloud_config['cache'].get( + 'max_age', self._cache_max_age) + self._cache_path = os.path.expanduser( + self.cloud_config['cache'].get('path', self._cache_path)) + def _load_config_file(self): for path in self._config_files: if os.path.exists(path): @@ -80,6 +91,12 @@ class OpenStackConfig(object): if os.path.exists(path): return yaml.load(open(path, 'r')) + def get_cache_max_age(self): + return self._cache_max_age + + def get_cache_path(self): + return self._cache_path + def _get_regions(self, cloud): try: return self.cloud_config['clouds'][cloud]['region_name'] From 0aa553c4735963f06e69ada0ba8568f6fff4701d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 10 Oct 2014 15:18:55 -0700 Subject: [PATCH 014/365] Handle the project/tenant nonesense more cleanly devstack clouds are more strict that public ones, so it's more important to get project_name vs. project_id correct. Solve it with brute force. Change-Id: I957b19c23266d379834361ab6a5b3b2dc5d15d3d --- os_client_config/config.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index f92c7909b..315d3e0ef 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -32,7 +32,7 @@ CACHE_PATH = os.path.join(os.path.expanduser( os.environ.get('XDG_CACHE_PATH', os.path.join('~', '.cache'))), 'openstack') BOOL_KEYS = ('insecure', 'cache') -REQUIRED_VALUES = ('auth_url', 'username', 'password', 'project_id') +REQUIRED_VALUES = ('auth_url', 'username', 'password') VENDOR_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] VENDOR_FILES = [ os.path.join(d, 'clouds-public.yaml') for d in VENDOR_SEARCH_PATH] @@ -54,7 +54,8 @@ class OpenStackConfig(object): defaults.add('username') defaults.add('user_domain_name') defaults.add('password') - defaults.add('project_id', defaults['username'], also='tenant_name') + defaults.add('project_name', defaults['username'], also='tenant_name') + defaults.add('project_id', also='tenant_id') defaults.add('project_domain_name') defaults.add('auth_url') defaults.add('region_name') @@ -136,6 +137,17 @@ class OpenStackConfig(object): cloud.update(our_cloud) if 'cloud' in cloud: del cloud['cloud'] + + return self._fix_project_madness(cloud) + + def _fix_project_madness(self, cloud): + project_name = None + # Do the list backwards so that project_name is the ultimate winner + for key in ('tenant_id', 'project_id', 'tenant_name', 'project_name'): + if key in cloud: + project_name = cloud[key] + del cloud[key] + cloud['project_name'] = project_name return cloud def get_all_clouds(self): @@ -166,6 +178,12 @@ class OpenStackConfig(object): ' config files {files}' ' or environment variables.'.format( name=name, files=','.join(self._config_files))) + if 'project_name' not in config and 'project_id' not in config: + raise exceptions.OpenStackConfigException( + 'Neither project_name or project_id information found' + ' for cloud {name} in config files {files}' + ' or environment variables.'.format( + name=name, files=','.join(self._config_files))) # If any of the defaults reference other values, we need to expand for (key, value) in config.items(): From 91afaeb4aeb811b28d27b9eaf3887235a841f680 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 10 Oct 2014 16:09:16 -0700 Subject: [PATCH 015/365] Handle lack of username for project_name defaults It's possible that there will not be a value in username, so we can't use it as a blind default. Change-Id: Iae93b9ec0e691c7b2174a0138c5455e36ad77ad7 --- os_client_config/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 315d3e0ef..781dc101b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -54,7 +54,9 @@ class OpenStackConfig(object): defaults.add('username') defaults.add('user_domain_name') defaults.add('password') - defaults.add('project_name', defaults['username'], also='tenant_name') + defaults.add( + 'project_name', defaults.get('username', None), + also='tenant_name') defaults.add('project_id', also='tenant_id') defaults.add('project_domain_name') defaults.add('auth_url') From b03d6e59b93cea29df30f286db02701e2e77effa Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 11 Oct 2014 09:41:48 -0700 Subject: [PATCH 016/365] Add support for command line argument processing Now takes the ability to pass in a dict of key/value pairs, probably from a command line processing thing like argparse, and to overlay them on the config that came from the files or env vars. Change-Id: I830699476e2340389979b34704c0dfbfe97a1e08 --- os_client_config/config.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 781dc101b..34de5134a 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -161,17 +161,43 @@ class OpenStackConfig(object): clouds.append(self.get_one_cloud(cloud, region)) return clouds - def get_one_cloud(self, name='openstack', region=None): + def _fix_args(self, args): + '''Replace - with _ and strip os_ prefixes.''' + os_args = dict() + new_args = dict() + for (key, val) in args.iteritems(): + key = key.replace('-', '_') + if key.startswith('os'): + os_args[key[3:]] = val + else: + new_args[key] = val + new_args.update(os_args) + return new_args - if not region: - region = self._get_region(name) + def get_one_cloud(self, **kwargs): + + args = self._fix_args(kwargs) + + if 'cloud' in args: + name = args['cloud'] + del args['cloud'] + else: + name = 'openstack' + + if 'region_name' not in args: + args['region_name'] = self._get_region(name) config = self._get_base_cloud_config(name) - config['region_name'] = region + + # Can't just do update, because None values take over + for (key, val) in args.iteritems(): + if val is not None: + config[key] = val for key in BOOL_KEYS: if key in config: - config[key] = get_boolean(config[key]) + if type(config[key]) is not bool: + config[key] = get_boolean(config[key]) for key in REQUIRED_VALUES: if key not in config or not config[key]: @@ -193,7 +219,7 @@ class OpenStackConfig(object): config[key] = value.format(**config) return cloud_config.CloudConfig( - name=name, region=region, config=config) + name=name, region=config['region_name'], config=config) if __name__ == '__main__': config = OpenStackConfig().get_all_clouds() From 470f6ea9358ac6cd6a05a96805d06c3cf8501d97 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Sat, 11 Oct 2014 14:55:30 -0500 Subject: [PATCH 017/365] Add support for argparse Namespace objects The Namespace objects returned by argparse contain all defined options even if they are unspecified and default to None or ''. Also it is not iterable. Change all that to add only the options presented to argparse to the cloud_config. Change-Id: Ia22fad60c81ab0b2878b404c0c8608d903ca964b --- os_client_config/config.py | 64 ++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 34de5134a..753c80aab 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -161,8 +161,24 @@ class OpenStackConfig(object): clouds.append(self.get_one_cloud(cloud, region)) return clouds - def _fix_args(self, args): - '''Replace - with _ and strip os_ prefixes.''' + def _fix_args(self, args, argparse=None): + """Massage the passed-in options + + Replace - with _ and strip os_ prefixes. + + Convert an argparse Namespace object to a dict, removing values + that are either None or ''. + """ + + if argparse: + # Convert the passed-in Namespace + o_dict = vars(argparse) + parsed_args = dict() + for k in o_dict: + if o_dict[k] is not None and o_dict[k] != '': + parsed_args[k] = o_dict[k] + args.update(parsed_args) + os_args = dict() new_args = dict() for (key, val) in args.iteritems(): @@ -174,13 +190,26 @@ class OpenStackConfig(object): new_args.update(os_args) return new_args - def get_one_cloud(self, **kwargs): + def get_one_cloud(self, cloud=None, validate=True, + argparse=None, **kwargs): + """Retrieve a single cloud configuration and merge additional options - args = self._fix_args(kwargs) + :param string cloud: + The name of the configuration to load from clouds.yaml + :param boolean validate: + Validate that required arguments are present and certain + argument combinations are valid + :param Namespace argparse: + An argparse Namespace object; allows direct passing in of + argparse options to be added to the cloud config. Values + of None and '' will be removed. + :param kwargs: Additional configuration options + """ - if 'cloud' in args: - name = args['cloud'] - del args['cloud'] + args = self._fix_args(kwargs, argparse=argparse) + + if cloud: + name = cloud else: name = 'openstack' @@ -199,19 +228,20 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) - for key in REQUIRED_VALUES: - if key not in config or not config[key]: + if validate: + for key in REQUIRED_VALUES: + if key not in config or not config[key]: + raise exceptions.OpenStackConfigException( + 'Unable to find full auth information for cloud' + ' {name} in config files {files}' + ' or environment variables.'.format( + name=name, files=','.join(self._config_files))) + if 'project_name' not in config and 'project_id' not in config: raise exceptions.OpenStackConfigException( - 'Unable to find full auth information for cloud {name} in' - ' config files {files}' + 'Neither project_name or project_id information found' + ' for cloud {name} in config files {files}' ' or environment variables.'.format( name=name, files=','.join(self._config_files))) - if 'project_name' not in config and 'project_id' not in config: - raise exceptions.OpenStackConfigException( - 'Neither project_name or project_id information found' - ' for cloud {name} in config files {files}' - ' or environment variables.'.format( - name=name, files=','.join(self._config_files))) # If any of the defaults reference other values, we need to expand for (key, value) in config.items(): From 0ed767e20937a62ed2f190c6645673236b11237a Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Sun, 12 Oct 2014 20:06:14 -0500 Subject: [PATCH 018/365] Map CloudConfig attributes to CloudConfig.config Treat the CloudConfig object as if it has the config attributes directly. And add some simple tests. This makes it easier to replace an argparse.Namespace() object with a CloudConfig object. It also might make initialization of some of the default attributes unnecessary. An example of this usage is in https://review.openstack.org/#/c/129795/1/openstackclient/shell.py Change-Id: I00ced540cf94742e8cb738f8f0767445ffeb4bfe --- os_client_config/cloud_config.py | 14 +++++++ os_client_config/tests/test_cloud_config.py | 44 +++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 os_client_config/tests/test_cloud_config.py diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index d0c932f41..c17bfc2b7 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -18,3 +18,17 @@ class CloudConfig(object): self.name = name self.region = region self.config = config + + def __getattr__(self, key): + """Return arbitrary attributes.""" + + if key.startswith('os_'): + key = key[3:] + + if key in [attr.replace('-', '_') for attr in self.config]: + return self.config[key] + else: + return None + + def __iter__(self): + return self.config.__iter__() diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py new file mode 100644 index 000000000..36386e50e --- /dev/null +++ b/os_client_config/tests/test_cloud_config.py @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from os_client_config import cloud_config +from os_client_config.tests import base + + +fake_config_dict = {'a': 1, 'os_b': 2, 'c': 3, 'os_c': 4} + + +class TestCloudConfig(base.TestCase): + + def test_arbitrary_attributes(self): + cc = cloud_config.CloudConfig("test1", "region-al", fake_config_dict) + self.assertEqual("test1", cc.name) + self.assertEqual("region-al", cc.region) + + # Look up straight value + self.assertEqual(1, cc.a) + + # Look up prefixed attribute, fail - returns None + self.assertEqual(None, cc.os_b) + + # Look up straight value, then prefixed value + self.assertEqual(3, cc.c) + self.assertEqual(3, cc.os_c) + + # Lookup mystery attribute + self.assertIsNone(cc.x) + + def test_iteration(self): + cc = cloud_config.CloudConfig("test1", "region-al", fake_config_dict) + self.assertTrue('a' in cc) + self.assertFalse('x' in cc) From 8f1bb101ea688456de7dcd269dc885558aaea383 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 26 Oct 2014 15:04:43 -0700 Subject: [PATCH 019/365] Fix a missed argument from a previous refactor Region name support got changed a little while ago, and a call that should now be a keyword argument style stayed as a positional ... which means that we lost region name support for clouds with more than one region. (it treated all of them like the first region) Change-Id: I666758a775b8fc8e03b7e9ddd3aa494c13505612 --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 753c80aab..925c23f48 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -158,7 +158,7 @@ class OpenStackConfig(object): for cloud in self._get_cloud_sections(): for region in self._get_regions(cloud).split(','): - clouds.append(self.get_one_cloud(cloud, region)) + clouds.append(self.get_one_cloud(cloud, region_name=region)) return clouds def _fix_args(self, args, argparse=None): From 1c025d548f63b613c198f47065bde8cf43655fb3 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 2 Nov 2014 13:44:36 +0100 Subject: [PATCH 020/365] Throw error if a non-existent cloud is requested The error messages when a bogus cloud is requested are very confusing. (They'll be things like "no auth url provided") Instead, be explicit on the problem. Change-Id: Idf68d1db7e5fccd712283775eb4d636d78ae5fc7 --- os_client_config/cloud_config.py | 2 +- os_client_config/config.py | 31 +++++++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index c17bfc2b7..6f501c692 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -15,7 +15,7 @@ class CloudConfig(object): def __init__(self, name, region, config): - self.name = name + self.name = name or 'openstack' self.region = region self.config = config diff --git a/os_client_config/config.py b/os_client_config/config.py index 925c23f48..ca3a75a73 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -115,10 +115,14 @@ class OpenStackConfig(object): def _get_base_cloud_config(self, name): cloud = dict() - if name in self.cloud_config['clouds']: - our_cloud = self.cloud_config['clouds'][name] - else: - our_cloud = dict() + + # Only validate cloud name if one was given + if name and name not in self.cloud_config['clouds']: + raise exceptions.OpenStackConfigException( + "Named cloud {name} requested that was not found.".format( + name=name)) + + our_cloud = self.cloud_config['clouds'].get(name, dict()) # Get the defaults (including env vars) first cloud.update(self.defaults) @@ -208,15 +212,10 @@ class OpenStackConfig(object): args = self._fix_args(kwargs, argparse=argparse) - if cloud: - name = cloud - else: - name = 'openstack' - if 'region_name' not in args: - args['region_name'] = self._get_region(name) + args['region_name'] = self._get_region(cloud) - config = self._get_base_cloud_config(name) + config = self._get_base_cloud_config(cloud) # Can't just do update, because None values take over for (key, val) in args.iteritems(): @@ -233,15 +232,15 @@ class OpenStackConfig(object): if key not in config or not config[key]: raise exceptions.OpenStackConfigException( 'Unable to find full auth information for cloud' - ' {name} in config files {files}' + ' {cloud} in config files {files}' ' or environment variables.'.format( - name=name, files=','.join(self._config_files))) + cloud=cloud, files=','.join(self._config_files))) if 'project_name' not in config and 'project_id' not in config: raise exceptions.OpenStackConfigException( 'Neither project_name or project_id information found' - ' for cloud {name} in config files {files}' + ' for cloud {cloud} in config files {files}' ' or environment variables.'.format( - name=name, files=','.join(self._config_files))) + cloud=cloud, files=','.join(self._config_files))) # If any of the defaults reference other values, we need to expand for (key, value) in config.items(): @@ -249,7 +248,7 @@ class OpenStackConfig(object): config[key] = value.format(**config) return cloud_config.CloudConfig( - name=name, region=config['region_name'], config=config) + name=cloud, region=config['region_name'], config=config) if __name__ == '__main__': config = OpenStackConfig().get_all_clouds() From 98d3de9f08886950dec4006fdce1b1d467a724d2 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 9 Nov 2014 08:18:42 -0300 Subject: [PATCH 021/365] Use yaml.safe_load instead of load. yaml.load will execute arbitrary code. Also use context managers to ensure files are closed Change-Id: I704baa7916ee834c12821009d8e3029b1b8fa340 --- os_client_config/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 925c23f48..c7119192b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -87,12 +87,14 @@ class OpenStackConfig(object): def _load_config_file(self): for path in self._config_files: if os.path.exists(path): - return yaml.load(open(path, 'r')) + with open(path, 'r') as f: + return yaml.safe_load(f) def _load_vendor_file(self): for path in self._vendor_files: if os.path.exists(path): - return yaml.load(open(path, 'r')) + with open(path, 'r') as f: + return yaml.safe_load(f) def get_cache_max_age(self): return self._cache_max_age From ca54d7290000f77c5382e227a7cf6e631a151174 Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Mon, 24 Nov 2014 17:10:43 +0000 Subject: [PATCH 022/365] Corrections to readme * README.rst: Remove docs and bugs links since the project is not currently publishing/consuming these anywhere that I can find. Also correct the source URL. Change-Id: I2b7002f7ea301a2c03a2e41e904de0c91a14831a --- README.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 30819f398..49814224f 100644 --- a/README.rst +++ b/README.rst @@ -143,6 +143,4 @@ Or, get all of the clouds. print(cloud.name, cloud.region, cloud.config) * Free software: Apache license -* Documentation: http://docs.openstack.org/developer/os-client-config -* Source: http://git.openstack.org/cgit/openstack/os-client-config -* Bugs: http://bugs.launchpad.net/os-client-config +* Source: http://git.openstack.org/cgit/stackforge/os-client-config From cdb3e37ccd079d10ca8fca334623a72fd260b726 Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Fri, 5 Dec 2014 03:30:46 +0000 Subject: [PATCH 023/365] Workflow documentation is now in infra-manual Replace URLs for workflow documentation to appropriate parts of the OpenStack Project Infrastructure Manual. Change-Id: I97d11d3a24a374c9212cd29f49472e199b1c8bc0 --- CONTRIBUTING.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c1f3dc297..1990ecf25 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,14 +1,13 @@ If you would like to contribute to the development of OpenStack, -you must follow the steps in the "If you're a developer, start here" -section of this page: +you must follow the steps in this page: - http://wiki.openstack.org/HowToContribute + http://docs.openstack.org/infra/manual/developers.html Once those steps have been completed, changes to OpenStack should be submitted for review via the Gerrit tool, following the workflow documented at: - http://wiki.openstack.org/GerritWorkflow + http://docs.openstack.org/infra/manual/developers.html#development-workflow Pull requests submitted through GitHub will be ignored. From 621a8636f505dd5c091a7fa8155ea79dc36b1b41 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 20 Jan 2015 18:04:21 -0800 Subject: [PATCH 024/365] Replace defaults_dict with scanning env vars All of the OS_ env vars start with OS_. Also, because of keystoneclient auth plugins, we have no idea which ones we need. Instead of positively identifying them - just grab them all - since they're all base layer and can be overridden by everything else anyway. Change-Id: I633f5e7d27c0a6a5c9b25f53cb99fe05b63c78ae --- os_client_config/config.py | 30 ++++++++++-------------------- os_client_config/defaults_dict.py | 27 --------------------------- 2 files changed, 10 insertions(+), 47 deletions(-) delete mode 100644 os_client_config/defaults_dict.py diff --git a/os_client_config/config.py b/os_client_config/config.py index 9656f6ca1..9c3ca874f 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -18,7 +18,6 @@ import os import yaml from os_client_config import cloud_config -from os_client_config import defaults_dict from os_client_config import exceptions from os_client_config import vendors @@ -44,31 +43,22 @@ def get_boolean(value): return False +def _get_os_environ(): + ret = dict() + for (k, v) in os.environ.items(): + if k.startswith('OS_'): + newkey = k[3:].lower() + ret[newkey] = v + return ret + + class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES - defaults = defaults_dict.DefaultsDict() - defaults.add('username') - defaults.add('user_domain_name') - defaults.add('password') - defaults.add( - 'project_name', defaults.get('username', None), - also='tenant_name') - defaults.add('project_id', also='tenant_id') - defaults.add('project_domain_name') - defaults.add('auth_url') - defaults.add('region_name') - defaults.add('cache') - defaults.add('auth_token') - defaults.add('insecure') - defaults.add('endpoint_type') - defaults.add('cacert') - defaults.add('auth_type') - - self.defaults = defaults + self.defaults = _get_os_environ() # use a config file if it exists where expected self.cloud_config = self._load_config_file() diff --git a/os_client_config/defaults_dict.py b/os_client_config/defaults_dict.py deleted file mode 100644 index 43de77beb..000000000 --- a/os_client_config/defaults_dict.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - - -class DefaultsDict(dict): - - def add(self, key, default_value=None, also=None, prefix=None): - if prefix: - key = '%s_%s' % (prefix.replace('-', '_'), key) - if also: - value = os.environ.get(also, default_value) - value = os.environ.get('OS_%s' % key.upper(), default_value) - if value is not None: - self.__setitem__(key, value) From c75daaa1f9f0881cae63337e0d9799555f569c06 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 20 Jan 2015 19:07:08 -0800 Subject: [PATCH 025/365] Support keystone auth plugins in a generic way. Basically, if keystoneclient is available, validate arguments into the auth dict. If it's not - we should probably be talking about what possible use this library has - but it should degrade cleanly and treat everything as passthrough. Change-Id: Ia31039d5c724eba22d053a004eefeaf6857f500d --- README.rst | 42 ++++++++++++------ os_client_config/config.py | 87 +++++++++++++++++++++++++++++-------- os_client_config/vendors.py | 9 +++- 3 files changed, 104 insertions(+), 34 deletions(-) diff --git a/README.rst b/README.rst index 30819f398..0d04f5f04 100644 --- a/README.rst +++ b/README.rst @@ -57,23 +57,27 @@ An example config file is probably helpful: clouds: mordred: cloud: hp - username: mordred@inaugust.com - password: XXXXXXXXX - project_id: mordred@inaugust.com + auth: + username: mordred@inaugust.com + password: XXXXXXXXX + project_name: mordred@inaugust.com region_name: region-b.geo-1 dns_service_type: hpext:dns + compute_api_version: 1.1 monty: - auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 - username: monty.taylor@hp.com - password: XXXXXXXX - project_id: monty.taylor@hp.com-default-tenant + auth: + auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 + username: monty.taylor@hp.com + password: XXXXXXXX + project_name: monty.taylor@hp.com-default-tenant region_name: region-b.geo-1 dns_service_type: hpext:dns infra: cloud: rackspace - username: openstackci - password: XXXXXXXX - project_id: 610275 + auth: + username: openstackci + password: XXXXXXXX + project_id: 610275 region_name: DFW,ORD,IAD You may note a few things. First, since auth_url settings are silly @@ -92,6 +96,17 @@ the setting with the default service type. That might strike you funny when setting `service_type` and it does me too - but that's just the world we live in. +Auth Settings +------------- + +Keystone has auth plugins - which means it's not possible to know ahead of time +which auth settings are needed. `os-client-config` sets the default plugin type +to `password`, which is what things all were before plugins came about. In +order to facilitate validation of values, all of the parameters that exist +as a result of a chosen plugin need to go into the auth dict. For password +auth, this includes `auth_url`, `username` and `password` as well as anything +related to domains, projects and trusts. + Cache Settings -------------- @@ -107,9 +122,10 @@ understands a simple set of cache control settings. clouds: mordred: cloud: hp - username: mordred@inaugust.com - password: XXXXXXXXX - project_id: mordred@inaugust.com + auth: + username: mordred@inaugust.com + password: XXXXXXXXX + project_name: mordred@inaugust.com region_name: region-b.geo-1 dns_service_type: hpext:dns diff --git a/os_client_config/config.py b/os_client_config/config.py index 9c3ca874f..67436fad9 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -17,6 +17,11 @@ import os import yaml +try: + import keystoneclient.auth as ksc_auth +except ImportError: + ksc_auth = None + from os_client_config import cloud_config from os_client_config import exceptions from os_client_config import vendors @@ -31,7 +36,6 @@ CACHE_PATH = os.path.join(os.path.expanduser( os.environ.get('XDG_CACHE_PATH', os.path.join('~', '.cache'))), 'openstack') BOOL_KEYS = ('insecure', 'cache') -REQUIRED_VALUES = ('auth_url', 'username', 'password') VENDOR_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] VENDOR_FILES = [ os.path.join(d, 'clouds-public.yaml') for d in VENDOR_SEARCH_PATH] @@ -44,7 +48,7 @@ def get_boolean(value): def _get_os_environ(): - ret = dict() + ret = dict(auth_plugin='password', auth=dict()) for (k, v) in os.environ.items(): if k.startswith('OS_'): newkey = k[3:].lower() @@ -52,6 +56,16 @@ def _get_os_environ(): return ret +def _auth_update(old_dict, new_dict): + """Like dict.update, except handling the nested dict called auth.""" + for (k, v) in new_dict.items(): + if k == 'auth': + old_dict[k].update(v) + else: + old_dict[k] = v + return old_dict + + class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None): @@ -124,15 +138,15 @@ class OpenStackConfig(object): cloud_name = our_cloud['cloud'] vendor_file = self._load_vendor_file() if vendor_file and cloud_name in vendor_file['public-clouds']: - cloud.update(vendor_file['public-clouds'][cloud_name]) + _auth_update(cloud, vendor_file['public-clouds'][cloud_name]) else: try: - cloud.update(vendors.CLOUD_DEFAULTS[cloud_name]) + _auth_update(cloud, vendors.CLOUD_DEFAULTS[cloud_name]) except KeyError: # Can't find the requested vendor config, go about business pass - cloud.update(our_cloud) + _auth_update(cloud, our_cloud) if 'cloud' in cloud: del cloud['cloud'] @@ -186,6 +200,53 @@ class OpenStackConfig(object): new_args.update(os_args) return new_args + def _find_winning_auth_value(self, opt, config): + opt_name = opt.name.replace('-', '_') + if opt_name in config: + return config[opt_name] + else: + for d_opt in opt.deprecated_opts: + d_opt_name = d_opt.name.replace('-', '_') + if d_opt_name in config: + return config[d_opt_name] + + def _validate_auth(self, config): + # May throw a keystoneclient.exceptions.NoMatchingPlugin + plugin_options = ksc_auth.get_plugin_class( + config['auth_plugin']).get_options() + + for p_opt in plugin_options: + # if it's in config.auth, win, kill it from config dict + # if it's in config and not in config.auth, move it + # deprecated loses to current + # provided beats default, deprecated or not + winning_value = self._find_winning_auth_value( + p_opt, config['auth']) + if not winning_value: + winning_value = self._find_winning_auth_value(p_opt, config) + + # if the plugin tells us that this value is required + # then error if it's doesn't exist now + if not winning_value and p_opt.required: + raise exceptions.OpenStackConfigException( + 'Unable to find auth information for cloud' + ' {cloud} in config files {files}' + ' or environment variables. Missing value {auth_key}' + ' required for auth plugin {plugin}'.format( + cloud=cloud, files=','.join(self._config_files), + auth_key=p_opt.name, plugin=config['auth_plugin'])) + + # Clean up after ourselves + for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: + opt = opt.replace('-', '_') + config.pop(opt, None) + config['auth'].pop(opt, None) + + if winning_value: + config['auth'][p_opt.name.replace('-', '_')] = winning_value + + return config + def get_one_cloud(self, cloud=None, validate=True, argparse=None, **kwargs): """Retrieve a single cloud configuration and merge additional options @@ -219,20 +280,8 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) - if validate: - for key in REQUIRED_VALUES: - if key not in config or not config[key]: - raise exceptions.OpenStackConfigException( - 'Unable to find full auth information for cloud' - ' {cloud} in config files {files}' - ' or environment variables.'.format( - cloud=cloud, files=','.join(self._config_files))) - if 'project_name' not in config and 'project_id' not in config: - raise exceptions.OpenStackConfigException( - 'Neither project_name or project_id information found' - ' for cloud {cloud} in config files {files}' - ' or environment variables.'.format( - cloud=cloud, files=','.join(self._config_files))) + if validate and ksc_auth: + config = self._validate_auth(config) # If any of the defaults reference other values, we need to expand for (key, value) in config.items(): diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index c78aaca54..eca437676 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -1,3 +1,4 @@ +# flake8: noqa # Copyright (c) 2014 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -14,12 +15,16 @@ CLOUD_DEFAULTS = dict( hp=dict( - auth_url='https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0', + auth=dict( + auth_url='https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0', + ), region_name='region-b.geo-1', dns_service_type='hpext:dns', ), rackspace=dict( - auth_url='https://identity.api.rackspacecloud.com/v2.0/', + auth=dict( + auth_url='https://identity.api.rackspacecloud.com/v2.0/', + ), database_service_type='rax:database', image_api_version='2', ) From bc5608837f7d144f2f480c45c9b5499fca22098e Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 20 Jan 2015 22:04:10 -0800 Subject: [PATCH 026/365] Start keeping default versions for all services It turns out we need to do the evil glance dance for almost everything. Change-Id: Ic0ad77ba0627bd4be88bdf0136aa04c2ba43afe6 --- os_client_config/config.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 67436fad9..bfc030efa 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -47,13 +47,12 @@ def get_boolean(value): return False -def _get_os_environ(): - ret = dict(auth_plugin='password', auth=dict()) +def _get_os_environ(defaults): for (k, v) in os.environ.items(): if k.startswith('OS_'): newkey = k[3:].lower() - ret[newkey] = v - return ret + defaults[newkey] = v + return defaults def _auth_update(old_dict, new_dict): @@ -72,7 +71,12 @@ class OpenStackConfig(object): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES - self.defaults = _get_os_environ() + defaults = dict( + auth_plugin='password', + auth=dict(), + compute_api_version='1.1', + ) + self.defaults = _get_os_environ(defaults) # use a config file if it exists where expected self.cloud_config = self._load_config_file() From 569197d763aec46f8fe9838e554082069c2e3cdc Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 27 Jan 2015 14:21:18 -0500 Subject: [PATCH 027/365] Provide Rackspace service_name override Rackspace puts two compute services in their catalog. This means that keystone session code cannot find the right one without a name match override. Change-Id: I1bc06b97261341ad01bf84ebf5a12294cd0d383c --- os_client_config/vendors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index eca437676..cab3676a1 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -26,6 +26,7 @@ CLOUD_DEFAULTS = dict( auth_url='https://identity.api.rackspacecloud.com/v2.0/', ), database_service_type='rax:database', + compute_service_name='cloudServersOpenStack', image_api_version='2', ) ) From 2bbcb630144975d032ecc312177a11266c2ac57b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 27 Jan 2015 16:51:53 -0500 Subject: [PATCH 028/365] Remove runtime depend on pbr os-client-config is not using pbr during runtime, so the runtime requirement for it was a lie. Change-Id: I3ed57ec5c2b0fdf4060ef17d6df1fa801cfa14cd --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9cf3f8ae0..71026aa67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,4 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. - -pbr>=0.6,!=0.7,<1.0 - PyYAML From 01d7728504a77bdf03d6e4aa98edd66d55fa13f0 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 11 Feb 2015 08:32:48 -0500 Subject: [PATCH 029/365] Make sure we're deep-copying the auth dict Having a defaults dict that has an empty dict and then does a bunch of updates means you have ONE instance of a dict that all of the other instances have references to. Change-Id: Id008f7ec98ff7b392553cebca5a5b301330e67a3 --- os_client_config/config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index bfc030efa..95dfaf7e1 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -59,7 +59,10 @@ def _auth_update(old_dict, new_dict): """Like dict.update, except handling the nested dict called auth.""" for (k, v) in new_dict.items(): if k == 'auth': - old_dict[k].update(v) + if k in old_dict: + old_dict[k].update(v) + else: + old_dict[k] = v.copy() else: old_dict[k] = v return old_dict @@ -73,7 +76,6 @@ class OpenStackConfig(object): defaults = dict( auth_plugin='password', - auth=dict(), compute_api_version='1.1', ) self.defaults = _get_os_environ(defaults) @@ -273,6 +275,8 @@ class OpenStackConfig(object): args['region_name'] = self._get_region(cloud) config = self._get_base_cloud_config(cloud) + if 'auth' not in config: + config['auth'] = dict() # Can't just do update, because None values take over for (key, val) in args.iteritems(): From 89d1e4d3c4cbbe67d60c5b96cfb8ae6762737fd2 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 11 Feb 2015 09:30:22 -0500 Subject: [PATCH 030/365] Don't return the auth dict inside the loop It turns out that when you're looping over a set of params to move them into an internal dict, returning inside the loop results in processing exactly one of them. This is not, it turns out, what you wanted. Change-Id: If1bf0c22b758e7238846b08991f4b0d25c841583 --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 95dfaf7e1..72c24f974 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -251,7 +251,7 @@ class OpenStackConfig(object): if winning_value: config['auth'][p_opt.name.replace('-', '_')] = winning_value - return config + return config def get_one_cloud(self, cloud=None, validate=True, argparse=None, **kwargs): From 56b75154fb4858fad5f1d7a910d5d7fbf785ef67 Mon Sep 17 00:00:00 2001 From: Julia Kreger Date: Sun, 8 Feb 2015 13:06:49 -0500 Subject: [PATCH 031/365] Allow region_name to be None When using the shade library with ansible playbooks and classical environment variables for authentication, it is possible for a region_name value to be passed in with the value of None. When the value is set to and preserved as None, the logic later on in the method will fail to create a config key/value entry for region_name which is required by code later on in the method. Modified the region_name check to not only check for the presence of a missing region_name value, but to check to see if that value is set to None. This allows the value to be reset preventing the method from failing on the return call due to a missing key. Change-Id: Id8a3edf53ac751f0c6ee4d71405a926ba90c0602 --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 72c24f974..ae19621ef 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -271,7 +271,7 @@ class OpenStackConfig(object): args = self._fix_args(kwargs, argparse=argparse) - if 'region_name' not in args: + if 'region_name' not in args or args['region_name'] is None: args['region_name'] = self._get_region(cloud) config = self._get_base_cloud_config(cloud) From 7385528671ea61617d7027fc33897ea850e7364c Mon Sep 17 00:00:00 2001 From: Julia Kreger Date: Wed, 11 Feb 2015 13:09:21 -0500 Subject: [PATCH 032/365] Prefer dest value when option is depricated Changed logic that re-assigns depricated key names while preserving their values, to prefer the value stored in the dest key if it exists, instead of attempting to generate the new key name. Change-Id: Ibe961688cdb6bd4c9b2dbd27b08c722c3c741586 --- os_client_config/config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index ae19621ef..f8adc047d 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -249,7 +249,13 @@ class OpenStackConfig(object): config['auth'].pop(opt, None) if winning_value: - config['auth'][p_opt.name.replace('-', '_')] = winning_value + # Prefer the plugin configuration dest value if the value's key + # is marked as depreciated. + if p_opt.dest is None: + config['auth'][p_opt.name.replace('-', '_')] = ( + winning_value) + else: + config['auth'][p_opt.dest] = winning_value return config From 076e9bd9bece61db43137eab553c1545e6701fca Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Thu, 26 Feb 2015 16:12:14 -0800 Subject: [PATCH 033/365] Add basic unit test for config Adding this exposed some python3 compatibility issues with iteritems. Change-Id: Ia78bd8edd17c7d2360ad958b3de734503d400774 --- os_client_config/config.py | 4 ++-- os_client_config/tests/test_config.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 os_client_config/tests/test_config.py diff --git a/os_client_config/config.py b/os_client_config/config.py index f8adc047d..7c50b5de4 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -197,7 +197,7 @@ class OpenStackConfig(object): os_args = dict() new_args = dict() - for (key, val) in args.iteritems(): + for (key, val) in iter(args.items()): key = key.replace('-', '_') if key.startswith('os'): os_args[key[3:]] = val @@ -285,7 +285,7 @@ class OpenStackConfig(object): config['auth'] = dict() # Can't just do update, because None values take over - for (key, val) in args.iteritems(): + for (key, val) in iter(args.items()): if val is not None: config[key] = val diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py new file mode 100644 index 000000000..2b1821eec --- /dev/null +++ b/os_client_config/tests/test_config.py @@ -0,0 +1,24 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +from os_client_config import cloud_config +from os_client_config import config + + +class TestConfig(testtools.TestCase): + def test_get_one_cloud(self): + c = config.OpenStackConfig() + self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) From 9752febeb6abd7d975af53e95ef1aed7c7d4cb56 Mon Sep 17 00:00:00 2001 From: Julia Kreger Date: Thu, 26 Feb 2015 13:14:17 -0500 Subject: [PATCH 034/365] Allow keystone validation bypass for noauth usage To enable the eventual use of the shade library for communication with ironic in situations where authentication is set to noauth, it is necessary to not attempt to validate a user's credentials. Added a check to disable authentication validation when the auth_plugin is set to '' or 'None' or None. Change-Id: I3807b4724ce5e204b5857c1dbf5325f0e3f4a78d --- os_client_config/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index f8adc047d..771a1c957 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -294,6 +294,10 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) + if 'auth_plugin' in config: + if config['auth_plugin'] in ('', 'None', None): + validate = False + if validate and ksc_auth: config = self._validate_auth(config) From e483abccbb84a9dd49b266105309610ca0924fb2 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Fri, 27 Feb 2015 11:23:16 -0800 Subject: [PATCH 035/365] More comprehensive unit tests for os-client-config This includes ensuring that yaml of a sane format can be loaded. Change-Id: I698b3139b7e44f000d2a413d17e79914ef542a22 --- os_client_config/tests/test_config.py | 41 +++++++++++++++++++++++++++ test-requirements.txt | 2 ++ 2 files changed, 43 insertions(+) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 2b1821eec..47e2b9a91 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -12,13 +12,54 @@ # License for the specific language governing permissions and limitations # under the License. +import tempfile + +import extras +import fixtures import testtools +import yaml from os_client_config import cloud_config from os_client_config import config class TestConfig(testtools.TestCase): + def get_config(self): + config = { + 'clouds': { + '_test_cloud_': { + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project_name': 'testproject', + }, + 'region_name': 'test-region', + }, + }, + 'cache': {'max_age': 1}, + } + tdir = self.useFixture(fixtures.TempDir()) + config['cache']['path'] = tdir.path + return config + def test_get_one_cloud(self): c = config.OpenStackConfig() self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) + + def test_get_one_cloud_with_config_files(self): + self.useFixture(fixtures.NestedTempfile()) + with tempfile.NamedTemporaryFile() as cloud_yaml: + cloud_yaml.write(yaml.safe_dump(self.get_config()).encode('utf-8')) + cloud_yaml.flush() + c = config.OpenStackConfig(config_files=[cloud_yaml.name]) + self.assertIsInstance(c.cloud_config, dict) + self.assertIn('cache', c.cloud_config) + self.assertIsInstance(c.cloud_config['cache'], dict) + self.assertIn('max_age', c.cloud_config['cache']) + self.assertIn('path', c.cloud_config['cache']) + cc = c.get_one_cloud('_test_cloud_') + self.assertIsInstance(cc, cloud_config.CloudConfig) + self.assertTrue(extras.safe_hasattr(cc, 'auth')) + self.assertIsInstance(cc.auth, dict) + self.assertIn('username', cc.auth) + self.assertEqual('testuser', cc.auth['username']) diff --git a/test-requirements.txt b/test-requirements.txt index d494076d9..03e01941e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,6 +5,8 @@ hacking>=0.9.2,<0.10 coverage>=3.6 +extras +fixtures>=0.3.14 discover python-subunit sphinx>=1.1.2 From 15cbdf7947d106c472181aa2ebc67a17974f54e3 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Fri, 27 Feb 2015 11:23:16 -0800 Subject: [PATCH 036/365] Add more testing of vendor yaml loading Adding coverage for vendor yaml loading and refactoring some tests to make the structure for that sane. Change-Id: I7aca0fcc0b04371f9a71e71c0224897b19cb04af --- os_client_config/tests/test_config.py | 76 +++++++++++++++++++-------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 47e2b9a91..b19899a5d 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -22,44 +22,76 @@ import yaml from os_client_config import cloud_config from os_client_config import config +VENDOR_CONF = { + 'public-clouds': { + '_test_cloud_in_our_cloud': { + 'auth': { + 'username': 'testotheruser', + 'project_name': 'testproject', + }, + }, + } +} +USER_CONF = { + 'clouds': { + '_test_cloud_': { + 'cloud': '_test_cloud_in_our_cloud', + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + }, + 'region_name': 'test-region', + }, + '_test_cloud_no_vendor': { + 'cloud': '_test_non_existant_cloud', + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project_name': 'testproject', + }, + 'region_name': 'test-region', + }, + }, + 'cache': {'max_age': 1}, +} + + +def _write_yaml(obj): + # Assume NestedTempfile so we don't have to cleanup + with tempfile.NamedTemporaryFile(delete=False) as obj_yaml: + obj_yaml.write(yaml.safe_dump(obj).encode('utf-8')) + return obj_yaml.name + class TestConfig(testtools.TestCase): - def get_config(self): - config = { - 'clouds': { - '_test_cloud_': { - 'auth': { - 'username': 'testuser', - 'password': 'testpass', - 'project_name': 'testproject', - }, - 'region_name': 'test-region', - }, - }, - 'cache': {'max_age': 1}, - } - tdir = self.useFixture(fixtures.TempDir()) - config['cache']['path'] = tdir.path - return config - def test_get_one_cloud(self): c = config.OpenStackConfig() self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) def test_get_one_cloud_with_config_files(self): self.useFixture(fixtures.NestedTempfile()) - with tempfile.NamedTemporaryFile() as cloud_yaml: - cloud_yaml.write(yaml.safe_dump(self.get_config()).encode('utf-8')) - cloud_yaml.flush() - c = config.OpenStackConfig(config_files=[cloud_yaml.name]) + conf = dict(USER_CONF) + tdir = self.useFixture(fixtures.TempDir()) + conf['cache']['path'] = tdir.path + cloud_yaml = _write_yaml(conf) + vendor_yaml = _write_yaml(VENDOR_CONF) + c = config.OpenStackConfig(config_files=[cloud_yaml], + vendor_files=[vendor_yaml]) self.assertIsInstance(c.cloud_config, dict) self.assertIn('cache', c.cloud_config) self.assertIsInstance(c.cloud_config['cache'], dict) self.assertIn('max_age', c.cloud_config['cache']) self.assertIn('path', c.cloud_config['cache']) cc = c.get_one_cloud('_test_cloud_') + self._assert_cloud_details(cc) + cc = c.get_one_cloud('_test_cloud_no_vendor') + self._assert_cloud_details(cc) + + def _assert_cloud_details(self, cc): self.assertIsInstance(cc, cloud_config.CloudConfig) self.assertTrue(extras.safe_hasattr(cc, 'auth')) self.assertIsInstance(cc.auth, dict) + self.assertIsNone(cc.cloud) self.assertIn('username', cc.auth) self.assertEqual('testuser', cc.auth['username']) + self.assertEqual('testproject', cc.auth['project_name']) From f99dc5f293bb71185be2102ea646b07159faf0a0 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Sat, 28 Feb 2015 07:45:01 -0800 Subject: [PATCH 037/365] Fix coverage report Coverage can't handle package names that have dashes, as it uses the package name to look for the base module name. So we need to pass in the base module name as it is imported. Change-Id: I2840eea85acaee2d05cab47fb67010e002a14bc0 --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 9be310a4c..860e3378b 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ commands = flake8 commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage --testr-args='{posargs}' +commands = python setup.py test --coverage --coverage-package-name=os_client_config --testr-args='{posargs}' [testenv:docs] commands = python setup.py build_sphinx @@ -31,4 +31,4 @@ commands = python setup.py build_sphinx show-source = True ignore = E123,E125,H803 builtins = _ -exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build \ No newline at end of file +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build From 1ccad8c41c9cab2f577e6505daab858a2c9d22b5 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 26 Feb 2015 08:57:46 -0500 Subject: [PATCH 038/365] Add support for configuring dogpile.cache shade has added dogpile.cache support. Since os-client-config already supports processing global cache settings, go ahead and add support for a couple of settings that can be used to feed dogpile.cache. Change-Id: I4d40753b83041c8a48b5c0a6d446f9e0de68220a --- os_client_config/config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index f8adc047d..83ec6c49d 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -88,11 +88,17 @@ class OpenStackConfig(object): self._cache_max_age = 300 self._cache_path = CACHE_PATH + self._cache_class = 'dogpile.cache.memory' + self._cache_arguments = {} if 'cache' in self.cloud_config: self._cache_max_age = self.cloud_config['cache'].get( 'max_age', self._cache_max_age) self._cache_path = os.path.expanduser( self.cloud_config['cache'].get('path', self._cache_path)) + self._cache_class = self.cloud_config['cache'].get( + 'class', self._cache_class) + self._cache_arguments = self.cloud_config['cache'].get( + 'arguments', self._cache_arguments) def _load_config_file(self): for path in self._config_files: @@ -112,6 +118,12 @@ class OpenStackConfig(object): def get_cache_path(self): return self._cache_path + def get_cache_class(self): + return self._cache_class + + def get_cache_arguments(self): + return self._cache_arguments + def _get_regions(self, cloud): try: return self.cloud_config['clouds'][cloud]['region_name'] From 6fff9fe2b41656739eca0a47d61e17e12e114f4f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 3 Mar 2015 07:31:32 -0500 Subject: [PATCH 039/365] Rename auth_plugin to auth_type To match with python-openstackclient is doing, change auth_plugin to auth_type. Since this is out in the wild already, add it to the backwards compat matrix. Change-Id: I36b3f18d57fa827028194f8af91ea309b53b6ee3 --- os_client_config/config.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 6bc84e26c..ecf463a5e 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -75,7 +75,7 @@ class OpenStackConfig(object): self._vendor_files = vendor_files or VENDOR_FILES defaults = dict( - auth_plugin='password', + auth_type='password', compute_api_version='1.1', ) self.defaults = _get_os_environ(defaults) @@ -168,16 +168,22 @@ class OpenStackConfig(object): if 'cloud' in cloud: del cloud['cloud'] - return self._fix_project_madness(cloud) + return self._fix_backwards_madness(cloud) - def _fix_project_madness(self, cloud): - project_name = None - # Do the list backwards so that project_name is the ultimate winner - for key in ('tenant_id', 'project_id', 'tenant_name', 'project_name'): - if key in cloud: - project_name = cloud[key] - del cloud[key] - cloud['project_name'] = project_name + def _fix_backwards_madness(self, cloud): + # Do the lists backwards so that project_name is the ultimate winner + mappings = { + 'project_name': ('tenant_id', 'project_id', + 'tenant_name', 'project_name'), + 'auth_type': ('auth_plugin', 'auth_type'), + } + for target_key, possible_values in mappings.items(): + target = None + for key in possible_values: + if key in cloud: + target = cloud[key] + del cloud[key] + cloud[target_key] = target return cloud def get_all_clouds(self): @@ -231,7 +237,7 @@ class OpenStackConfig(object): def _validate_auth(self, config): # May throw a keystoneclient.exceptions.NoMatchingPlugin plugin_options = ksc_auth.get_plugin_class( - config['auth_plugin']).get_options() + config['auth_type']).get_options() for p_opt in plugin_options: # if it's in config.auth, win, kill it from config dict @@ -252,7 +258,7 @@ class OpenStackConfig(object): ' or environment variables. Missing value {auth_key}' ' required for auth plugin {plugin}'.format( cloud=cloud, files=','.join(self._config_files), - auth_key=p_opt.name, plugin=config['auth_plugin'])) + auth_key=p_opt.name, plugin=config.get('auth_type'))) # Clean up after ourselves for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: @@ -306,8 +312,8 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) - if 'auth_plugin' in config: - if config['auth_plugin'] in ('', 'None', None): + if 'auth_type' in config: + if config['auth_type'] in ('', 'None', None): validate = False if validate and ksc_auth: From df91dcf111a7aeecad948481fc102bedd3fad05d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 3 Mar 2015 16:34:19 -0500 Subject: [PATCH 040/365] Add two newlines to the ends of files flake8 bites it for me on these locally. Change-Id: I99291c64fafff423aa720da7dba030526ca0cb50 --- os_client_config/tests/base.py | 2 +- os_client_config/tests/test_os_client_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 185fd6fd9..1c30cdb56 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -20,4 +20,4 @@ from oslotest import base class TestCase(base.BaseTestCase): - """Test case base class for all unit tests.""" \ No newline at end of file + """Test case base class for all unit tests.""" diff --git a/os_client_config/tests/test_os_client_config.py b/os_client_config/tests/test_os_client_config.py index 9b26ad547..7421b6fe4 100644 --- a/os_client_config/tests/test_os_client_config.py +++ b/os_client_config/tests/test_os_client_config.py @@ -25,4 +25,4 @@ from os_client_config.tests import base class TestOs_client_config(base.TestCase): def test_something(self): - pass \ No newline at end of file + pass From 3b5a20ea037c8e01f5da2f98e9e47082cefad89e Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 3 Mar 2015 16:09:00 -0500 Subject: [PATCH 041/365] Handle project_name/tenant_name in the auth dict We were doing backwards compat for project/tenant in a way that didn't notice anything in the auth dict - which is there project/tenant info goes. Ooops. Change-Id: I141c1d99f31f381898bf993c4e7fcab1078f40c6 --- os_client_config/config.py | 26 ++++++++++++++++++++++++-- os_client_config/tests/test_config.py | 22 +++++++++++++--------- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index ecf463a5e..3565fa684 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -164,6 +164,9 @@ class OpenStackConfig(object): # Can't find the requested vendor config, go about business pass + if 'auth' not in cloud: + cloud['auth'] = dict() + _auth_update(cloud, our_cloud) if 'cloud' in cloud: del cloud['cloud'] @@ -171,10 +174,31 @@ class OpenStackConfig(object): return self._fix_backwards_madness(cloud) def _fix_backwards_madness(self, cloud): + cloud = self._fix_backwards_project(cloud) + cloud = self._fix_backwards_auth_plugin(cloud) + return cloud + + def _fix_backwards_project(self, cloud): # Do the lists backwards so that project_name is the ultimate winner mappings = { 'project_name': ('tenant_id', 'project_id', 'tenant_name', 'project_name'), + } + for target_key, possible_values in mappings.items(): + target = None + for key in possible_values: + if key in cloud: + target = cloud[key] + del cloud[key] + if key in cloud['auth']: + target = cloud['auth'][key] + del cloud['auth'][key] + cloud['auth'][target_key] = target + return cloud + + def _fix_backwards_auth_plugin(self, cloud): + # Do the lists backwards so that auth_type is the ultimate winner + mappings = { 'auth_type': ('auth_plugin', 'auth_type'), } for target_key, possible_values in mappings.items(): @@ -299,8 +323,6 @@ class OpenStackConfig(object): args['region_name'] = self._get_region(cloud) config = self._get_base_cloud_config(cloud) - if 'auth' not in config: - config['auth'] = dict() # Can't just do update, because None values take over for (key, val) in iter(args.items()): diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index b19899a5d..5ef3048f0 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -64,19 +64,23 @@ def _write_yaml(obj): class TestConfig(testtools.TestCase): - def test_get_one_cloud(self): - c = config.OpenStackConfig() - self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) - - def test_get_one_cloud_with_config_files(self): + def setUp(self): + super(TestConfig, self).setUp() self.useFixture(fixtures.NestedTempfile()) conf = dict(USER_CONF) tdir = self.useFixture(fixtures.TempDir()) conf['cache']['path'] = tdir.path - cloud_yaml = _write_yaml(conf) - vendor_yaml = _write_yaml(VENDOR_CONF) - c = config.OpenStackConfig(config_files=[cloud_yaml], - vendor_files=[vendor_yaml]) + self.cloud_yaml = _write_yaml(conf) + self.vendor_yaml = _write_yaml(VENDOR_CONF) + + def test_get_one_cloud(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) + + def test_get_one_cloud_with_config_files(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) self.assertIsInstance(c.cloud_config, dict) self.assertIn('cache', c.cloud_config) self.assertIsInstance(c.cloud_config['cache'], dict) From d5931d4658b43b5451adc38be5d6b7f257291098 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 5 Mar 2015 08:51:18 -0500 Subject: [PATCH 042/365] Flesh out api version defaults openstackclient needs a bit richer support for api version defaults. Namely, it knows what defaults it wants to have - but we need to do defaults processing in os-client-config to get sequencing correct. So provide an API call to set new defaults that can be used before config processing. Also, flesh out the dict of known default values with good defaults to match osc behavior, and add the known v1 default of HP to the vendors.py values. Change-Id: I45e2550af58aee616ca168d20a557077beeab007 --- os_client_config/config.py | 18 ++++++++++-------- os_client_config/defaults.py | 23 +++++++++++++++++++++++ os_client_config/vendors.py | 1 + 3 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 os_client_config/defaults.py diff --git a/os_client_config/config.py b/os_client_config/config.py index 3565fa684..48b99f591 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -23,6 +23,7 @@ except ImportError: ksc_auth = None from os_client_config import cloud_config +from os_client_config import defaults from os_client_config import exceptions from os_client_config import vendors @@ -41,18 +42,23 @@ VENDOR_FILES = [ os.path.join(d, 'clouds-public.yaml') for d in VENDOR_SEARCH_PATH] +def set_default(key, value): + defaults._defaults[key] = value + + def get_boolean(value): if value.lower() == 'true': return True return False -def _get_os_environ(defaults): +def _get_os_environ(): + ret = dict(defaults._defaults) for (k, v) in os.environ.items(): if k.startswith('OS_'): newkey = k[3:].lower() - defaults[newkey] = v - return defaults + ret[newkey] = v + return ret def _auth_update(old_dict, new_dict): @@ -74,11 +80,7 @@ class OpenStackConfig(object): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES - defaults = dict( - auth_type='password', - compute_api_version='1.1', - ) - self.defaults = _get_os_environ(defaults) + self.defaults = _get_os_environ() # use a config file if it exists where expected self.cloud_config = self._load_config_file() diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py new file mode 100644 index 000000000..5f70d6ee4 --- /dev/null +++ b/os_client_config/defaults.py @@ -0,0 +1,23 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +_defaults = dict( + auth_type='password', + compute_api_version='2', + identity_api_version='2', + image_api_version='1', + network_api_version='2', + object_api_version='1', + volume_api_version='1', +) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index cab3676a1..eb08bbdbd 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -20,6 +20,7 @@ CLOUD_DEFAULTS = dict( ), region_name='region-b.geo-1', dns_service_type='hpext:dns', + image_api_version='1', ), rackspace=dict( auth=dict( From e5d2782cef302dbc677ebba06f5f7f3e520b1031 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Tue, 10 Mar 2015 19:00:50 +0100 Subject: [PATCH 043/365] Add cover to .gitignore Coverage reports in the cover directory should be ignored by git Change-Id: I4cde6b0b8fa2fa04ce6a4c308af3c37dfcdf1b5d --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e6a97ec61..837459343 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ lib64 pip-log.txt # Unit test / coverage reports +cover .coverage .tox nosetests.xml @@ -49,4 +50,4 @@ ChangeLog # Editors *~ .*.swp -.*sw? \ No newline at end of file +.*sw? From 63e1630f2b44e1f867ed887ea7d66b76daca671a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 6 Mar 2015 08:21:43 -0500 Subject: [PATCH 044/365] Change dogpile cache defaults Memory cache can grow unbounded, so it should really be opt in. Change to match shade with the following defaults: - If you specify nothing, you get null cache - If you specify an expiration time and nothing else, you get memory cache - If you specify an explicit cache class, you will get that class Change-Id: I6c9eab71a88a534de7e52ad2a564450c44aacc1d --- README.rst | 14 +++++++++++--- os_client_config/config.py | 6 ++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index e6dbed072..4c67acfab 100644 --- a/README.rst +++ b/README.rst @@ -112,13 +112,21 @@ Cache Settings Accessing a cloud is often expensive, so it's quite common to want to do some client-side caching of those operations. To facilitate that, os-client-config -understands a simple set of cache control settings. +understands passing through cache settings to dogpile.cache, with the following +behaviors: + +* Listing no config settings means you get a null cache. +* `cache.max_age` and nothing else gets you memory cache. +* Otherwise, `cache.class` and `cache.arguments` are passed in :: cache: - path: ~/.cache/openstack - max_age: 300 + class: dogpile.cache.pylibmc + max_age: 3600 + arguments: + url: + - 127.0.0.1 clouds: mordred: cloud: hp diff --git a/os_client_config/config.py b/os_client_config/config.py index 48b99f591..84940436e 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -88,13 +88,15 @@ class OpenStackConfig(object): self.cloud_config = dict( clouds=dict(openstack=dict(self.defaults))) - self._cache_max_age = 300 + self._cache_max_age = None self._cache_path = CACHE_PATH - self._cache_class = 'dogpile.cache.memory' + self._cache_class = 'dogpile.cache.null' self._cache_arguments = {} if 'cache' in self.cloud_config: self._cache_max_age = self.cloud_config['cache'].get( 'max_age', self._cache_max_age) + if self._cache_max_age: + self._cache_class = 'dogpile.cache.memory' self._cache_path = os.path.expanduser( self.cloud_config['cache'].get('path', self._cache_path)) self._cache_class = self.cloud_config['cache'].get( From 79a1c489a2fa72ee26f7434dd8b443b69c403a4d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 28 Mar 2015 11:36:44 -0400 Subject: [PATCH 045/365] Update .gitreview for git section rename Change-Id: I4a19b251c5fdf76ffb10387002a13336203a7b72 --- .gitreview | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitreview b/.gitreview index 234652006..5ba7eddc1 100644 --- a/.gitreview +++ b/.gitreview @@ -1,4 +1,4 @@ [gerrit] host=review.openstack.org port=29418 -project=stackforge/os-client-config.git +project=openstack/os-client-config.git From 52e676644415a7dd14629b3bdd467d9139850aa6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 11 Apr 2015 08:06:29 -0400 Subject: [PATCH 046/365] Actually put some content into our sphinx docs The README is pretty good - get it into the main index page for the sphinx docs so that we can publish them. Partial-Bug: #1440814 Change-Id: Ic72b81964cab1f939f08b957dec3be969c47a32e --- doc/source/conf.py | 4 ++-- doc/source/index.rst | 14 ++------------ doc/source/readme.rst | 1 - doc/source/usage.rst | 7 ------- 4 files changed, 4 insertions(+), 22 deletions(-) delete mode 100644 doc/source/readme.rst delete mode 100644 doc/source/usage.rst diff --git a/doc/source/conf.py b/doc/source/conf.py index 02be16776..221de3c88 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,7 +38,7 @@ master_doc = 'index' # General information about the project. project = u'os-client-config' -copyright = u'2013, OpenStack Foundation' +copyright = u'2015, various OpenStack developers' # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = True @@ -72,4 +72,4 @@ latex_documents = [ ] # Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'http://docs.python.org/': None} \ No newline at end of file +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/index.rst b/doc/source/index.rst index 2c8b52d5e..efde7601a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,20 +1,10 @@ -.. os-client-config documentation master file, created by - sphinx-quickstart on Tue Jul 9 22:26:36 2013. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to os-client-config's documentation! -======================================================== - -Contents: +.. include:: ../../README.rst .. toctree:: :maxdepth: 2 - readme - installation - usage contributing + installation Indices and tables ================== diff --git a/doc/source/readme.rst b/doc/source/readme.rst deleted file mode 100644 index 38ba8043d..000000000 --- a/doc/source/readme.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../README.rst \ No newline at end of file diff --git a/doc/source/usage.rst b/doc/source/usage.rst deleted file mode 100644 index 910fd0790..000000000 --- a/doc/source/usage.rst +++ /dev/null @@ -1,7 +0,0 @@ -======== -Usage -======== - -To use os-client-config in a project:: - - import os_client_config \ No newline at end of file From 2ac9258563a969283472fbde053850eccbc2fbf2 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 11 Apr 2015 11:32:33 -0400 Subject: [PATCH 047/365] Add keystoneclient to test-requirements The auth parameter name validation requires keystoneclient and can't be tested if it's not there. While we're at it - update the current requirements to be inline with global requirements. Change-Id: I6da62476f3851670545143184f9f29479f1caaca --- requirements.txt | 2 +- test-requirements.txt | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 71026aa67..498c5c336 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -PyYAML +PyYAML>=3.1.0 diff --git a/test-requirements.txt b/test-requirements.txt index 03e01941e..62f3e8831 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,10 +8,11 @@ coverage>=3.6 extras fixtures>=0.3.14 discover -python-subunit -sphinx>=1.1.2 -oslosphinx -oslotest>=1.1.0.0a1 +python-keystoneclient>=1.1.0 +python-subunit>=0.0.18 +sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 +oslosphinx>=2.5.0,<2.6.0 # Apache-2.0 +oslotest>=1.5.1,<1.6.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4 -testtools>=0.9.34 +testtools>=0.9.36,!=1.2.0 From 7e682d3bf097a006ec43c16ecc96664bf4b29294 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 11 Apr 2015 08:27:05 -0400 Subject: [PATCH 048/365] Put env vars into their own cloud config The semantics around mixing environment variables and config file values are confusing at best and no reasonable usecase has been expressed as to why doing so is desirable. Move the logic around environment variable processing to always provide an "envvars" cloud if any envvars are set. The cloud will only exist in the presence of OS_ env vars. get_one_cloud() will default to returning the envvars cloud if it exists. Change-Id: I6c3a54997c3278feedfdf93cc4d1e74b6235700a Closes-Bug: #1439927 --- README.rst | 15 ++-- os_client_config/cloud_config.py | 2 +- os_client_config/config.py | 31 +++++-- os_client_config/tests/base.py | 80 ++++++++++++++++++- os_client_config/tests/test_config.py | 67 +--------------- os_client_config/tests/test_environ.py | 67 ++++++++++++++++ .../tests/test_os_client_config.py | 28 ------- 7 files changed, 181 insertions(+), 109 deletions(-) create mode 100644 os_client_config/tests/test_environ.py delete mode 100644 os_client_config/tests/test_os_client_config.py diff --git a/README.rst b/README.rst index 4c67acfab..18bf81d77 100644 --- a/README.rst +++ b/README.rst @@ -16,14 +16,13 @@ os-client-config honors all of the normal `OS_*` variables. It does not provide backwards compatibility to service-specific variables such as `NOVA_USERNAME`. -If you have environment variables and no config files, os-client-config -will produce a cloud config object named "openstack" containing your -values from the environment. +If you have OpenStack environment variables seet and no config files, +os-client-config will produce a cloud config object named "envvars" containing +your values from the environment. Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type -for trove (because you're using Rackspace) set: -:: +for trove set:: export OS_DATABASE_SERVICE_TYPE=rax:database @@ -40,7 +39,7 @@ locations: The first file found wins. The keys are all of the keys you'd expect from `OS_*` - except lower case -and without the OS prefix. So, username is set with `username`. +and without the OS prefix. So, region name is set with `region_name`. Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type @@ -119,6 +118,10 @@ behaviors: * `cache.max_age` and nothing else gets you memory cache. * Otherwise, `cache.class` and `cache.arguments` are passed in +`os-client-config` does not actually cache anything itself, but it collects +and presents the cache information so that your various applications that +are connecting to OpenStack can share a cache should you desire. + :: cache: diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 6f501c692..c17bfc2b7 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -15,7 +15,7 @@ class CloudConfig(object): def __init__(self, name, region, config): - self.name = name or 'openstack' + self.name = name self.region = region self.config = config diff --git a/os_client_config/config.py b/os_client_config/config.py index 84940436e..70326cfc0 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -54,10 +54,12 @@ def get_boolean(value): def _get_os_environ(): ret = dict(defaults._defaults) - for (k, v) in os.environ.items(): - if k.startswith('OS_'): - newkey = k[3:].lower() - ret[newkey] = v + environkeys = [k for k in os.environ.keys() if k.startswith('OS_')] + if not environkeys: + return None + for k in environkeys: + newkey = k[3:].lower() + ret[newkey] = os.environ[k] return ret @@ -80,7 +82,7 @@ class OpenStackConfig(object): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES - self.defaults = _get_os_environ() + self.defaults = dict(defaults._defaults) # use a config file if it exists where expected self.cloud_config = self._load_config_file() @@ -88,6 +90,14 @@ class OpenStackConfig(object): self.cloud_config = dict( clouds=dict(openstack=dict(self.defaults))) + envvars = _get_os_environ() + if envvars: + if 'envvars' in self.cloud_config['clouds']: + raise exceptions.OpenStackConfigException( + 'clouds.yaml defines a cloud named envvars, and OS_' + ' env vars are set') + self.cloud_config['clouds']['envvars'] = envvars + self._cache_max_age = None self._cache_path = CACHE_PATH self._cache_class = 'dogpile.cache.null' @@ -109,6 +119,7 @@ class OpenStackConfig(object): if os.path.exists(path): with open(path, 'r') as f: return yaml.safe_load(f) + return dict(clouds=dict()) def _load_vendor_file(self): for path in self._vendor_files: @@ -135,7 +146,7 @@ class OpenStackConfig(object): # No region configured return '' - def _get_region(self, cloud): + def _get_region(self, cloud=None): return self._get_regions(cloud).split(',')[0] def _get_cloud_sections(self): @@ -152,7 +163,7 @@ class OpenStackConfig(object): our_cloud = self.cloud_config['clouds'].get(name, dict()) - # Get the defaults (including env vars) first + # Get the defaults cloud.update(self.defaults) # yes, I know the next line looks silly @@ -197,7 +208,8 @@ class OpenStackConfig(object): if key in cloud['auth']: target = cloud['auth'][key] del cloud['auth'][key] - cloud['auth'][target_key] = target + if target: + cloud['auth'][target_key] = target return cloud def _fix_backwards_auth_plugin(self, cloud): @@ -321,6 +333,9 @@ class OpenStackConfig(object): :param kwargs: Additional configuration options """ + if cloud is None and 'envvars' in self._get_cloud_sections(): + cloud = 'envvars' + args = self._fix_args(kwargs, argparse=argparse) if 'region_name' not in args or args['region_name'] is None: diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 1c30cdb56..f6e815d9c 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -15,9 +15,87 @@ # License for the specific language governing permissions and limitations # under the License. + +import os +import tempfile + +from os_client_config import cloud_config + +import extras +import fixtures from oslotest import base +import yaml + + +VENDOR_CONF = { + 'public-clouds': { + '_test_cloud_in_our_cloud': { + 'auth': { + 'username': 'testotheruser', + 'project_name': 'testproject', + }, + }, + } +} +USER_CONF = { + 'clouds': { + '_test_cloud_': { + 'cloud': '_test_cloud_in_our_cloud', + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + }, + 'region_name': 'test-region', + }, + '_test_cloud_no_vendor': { + 'cloud': '_test_non_existant_cloud', + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project_name': 'testproject', + }, + 'region_name': 'test-region', + }, + }, + 'cache': {'max_age': 1}, +} + + +def _write_yaml(obj): + # Assume NestedTempfile so we don't have to cleanup + with tempfile.NamedTemporaryFile(delete=False) as obj_yaml: + obj_yaml.write(yaml.safe_dump(obj).encode('utf-8')) + return obj_yaml.name class TestCase(base.BaseTestCase): - """Test case base class for all unit tests.""" + + def setUp(self): + super(TestCase, self).setUp() + + self.useFixture(fixtures.NestedTempfile()) + conf = dict(USER_CONF) + tdir = self.useFixture(fixtures.TempDir()) + conf['cache']['path'] = tdir.path + self.cloud_yaml = _write_yaml(conf) + self.vendor_yaml = _write_yaml(VENDOR_CONF) + + # Isolate the test runs from the environment + # Do this as two loops because you can't modify the dict in a loop + # over the dict in 3.4 + keys_to_isolate = [] + for env in os.environ.keys(): + if env.startswith('OS_'): + keys_to_isolate.append(env) + for env in keys_to_isolate: + self.useFixture(fixtures.EnvironmentVariable(env)) + + def _assert_cloud_details(self, cc): + self.assertIsInstance(cc, cloud_config.CloudConfig) + self.assertTrue(extras.safe_hasattr(cc, 'auth')) + self.assertIsInstance(cc.auth, dict) + self.assertIsNone(cc.cloud) + self.assertIn('username', cc.auth) + self.assertEqual('testuser', cc.auth['username']) + self.assertEqual('testproject', cc.auth['project_name']) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 5ef3048f0..9decf3efa 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -12,66 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -import tempfile - -import extras -import fixtures -import testtools -import yaml - from os_client_config import cloud_config from os_client_config import config - -VENDOR_CONF = { - 'public-clouds': { - '_test_cloud_in_our_cloud': { - 'auth': { - 'username': 'testotheruser', - 'project_name': 'testproject', - }, - }, - } -} -USER_CONF = { - 'clouds': { - '_test_cloud_': { - 'cloud': '_test_cloud_in_our_cloud', - 'auth': { - 'username': 'testuser', - 'password': 'testpass', - }, - 'region_name': 'test-region', - }, - '_test_cloud_no_vendor': { - 'cloud': '_test_non_existant_cloud', - 'auth': { - 'username': 'testuser', - 'password': 'testpass', - 'project_name': 'testproject', - }, - 'region_name': 'test-region', - }, - }, - 'cache': {'max_age': 1}, -} +from os_client_config.tests import base -def _write_yaml(obj): - # Assume NestedTempfile so we don't have to cleanup - with tempfile.NamedTemporaryFile(delete=False) as obj_yaml: - obj_yaml.write(yaml.safe_dump(obj).encode('utf-8')) - return obj_yaml.name - - -class TestConfig(testtools.TestCase): - def setUp(self): - super(TestConfig, self).setUp() - self.useFixture(fixtures.NestedTempfile()) - conf = dict(USER_CONF) - tdir = self.useFixture(fixtures.TempDir()) - conf['cache']['path'] = tdir.path - self.cloud_yaml = _write_yaml(conf) - self.vendor_yaml = _write_yaml(VENDOR_CONF) +class TestConfig(base.TestCase): def test_get_one_cloud(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], @@ -90,12 +36,3 @@ class TestConfig(testtools.TestCase): self._assert_cloud_details(cc) cc = c.get_one_cloud('_test_cloud_no_vendor') self._assert_cloud_details(cc) - - def _assert_cloud_details(self, cc): - self.assertIsInstance(cc, cloud_config.CloudConfig) - self.assertTrue(extras.safe_hasattr(cc, 'auth')) - self.assertIsInstance(cc.auth, dict) - self.assertIsNone(cc.cloud) - self.assertIn('username', cc.auth) - self.assertEqual('testuser', cc.auth['username']) - self.assertEqual('testproject', cc.auth['project_name']) diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py new file mode 100644 index 000000000..473c41743 --- /dev/null +++ b/os_client_config/tests/test_environ.py @@ -0,0 +1,67 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from os_client_config import cloud_config +from os_client_config import config +from os_client_config import exceptions +from os_client_config.tests import base + +import fixtures + + +class TestConfig(base.TestCase): + + def test_get_one_cloud(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) + + def test_no_environ(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + self.assertRaises( + exceptions.OpenStackConfigException, c.get_one_cloud, 'envvars') + + def test_environ_exists(self): + self.useFixture( + fixtures.EnvironmentVariable('OS_AUTH_URL', 'https://example.com')) + self.useFixture( + fixtures.EnvironmentVariable('OS_USERNAME', 'testuser')) + self.useFixture( + fixtures.EnvironmentVariable('OS_PROJECT_NAME', 'testproject')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('envvars') + self._assert_cloud_details(cc) + self.assertNotIn('auth_url', cc.config) + self.assertIn('auth_url', cc.config['auth']) + self.assertNotIn('auth_url', cc.config) + cc = c.get_one_cloud('_test_cloud_') + self._assert_cloud_details(cc) + cc = c.get_one_cloud('_test_cloud_no_vendor') + self._assert_cloud_details(cc) + + def test_get_one_cloud_with_config_files(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + self.assertIsInstance(c.cloud_config, dict) + self.assertIn('cache', c.cloud_config) + self.assertIsInstance(c.cloud_config['cache'], dict) + self.assertIn('max_age', c.cloud_config['cache']) + self.assertIn('path', c.cloud_config['cache']) + cc = c.get_one_cloud('_test_cloud_') + self._assert_cloud_details(cc) + cc = c.get_one_cloud('_test_cloud_no_vendor') + self._assert_cloud_details(cc) diff --git a/os_client_config/tests/test_os_client_config.py b/os_client_config/tests/test_os_client_config.py deleted file mode 100644 index 7421b6fe4..000000000 --- a/os_client_config/tests/test_os_client_config.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -test_os_client_config ----------------------------------- - -Tests for `os_client_config` module. -""" - -from os_client_config.tests import base - - -class TestOs_client_config(base.TestCase): - - def test_something(self): - pass From ffafb52fa7d41e7e0d4d3a44588d94dcc8dfa200 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 11 Apr 2015 11:55:52 -0400 Subject: [PATCH 049/365] Allow overriding envvars as the name of the cloud For environment variables created cloud objects, it's possible someone may not want it to be called envvars. I mean, let's be honest, I cannot imagine why this would be important ... but people get emotional about things. Let them name their cloud "bunnyrabbit" because that makes people happy. Change-Id: I0c232de7d93080ea632fb66a82b9e6d3e925c901 --- README.rst | 3 ++- os_client_config/config.py | 21 +++++++++++++---- os_client_config/tests/test_config.py | 7 ++++++ os_client_config/tests/test_environ.py | 32 +++++++++++++++----------- 4 files changed, 43 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 18bf81d77..34a1b02d8 100644 --- a/README.rst +++ b/README.rst @@ -18,7 +18,8 @@ provide backwards compatibility to service-specific variables such as If you have OpenStack environment variables seet and no config files, os-client-config will produce a cloud config object named "envvars" containing -your values from the environment. +your values from the environment. If you don't like the name "envvars", that's +ok, you can override it by setting `OS_CLOUD_NAME`. Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type diff --git a/os_client_config/config.py b/os_client_config/config.py index 70326cfc0..a0e7672a1 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -90,13 +90,24 @@ class OpenStackConfig(object): self.cloud_config = dict( clouds=dict(openstack=dict(self.defaults))) + self.envvar_key = os.environ.pop('OS_CLOUD_NAME', None) + if self.envvar_key: + if self.envvar_key in self.cloud_config['clouds']: + raise exceptions.OpenStackConfigException( + 'clouds.yaml defines a cloud named "{0}", but' + ' OS_CLOUD_NAME is also set to "{0}". Please rename' + ' either your environment based cloud, or one of your' + ' file-based clouds.'.format(self.envvar_key)) + else: + self.envvar_key = 'envvars' + envvars = _get_os_environ() if envvars: - if 'envvars' in self.cloud_config['clouds']: + if self.envvar_key in self.cloud_config['clouds']: raise exceptions.OpenStackConfigException( - 'clouds.yaml defines a cloud named envvars, and OS_' + 'clouds.yaml defines a cloud named {0}, and OS_*' ' env vars are set') - self.cloud_config['clouds']['envvars'] = envvars + self.cloud_config['clouds'][self.envvar_key] = envvars self._cache_max_age = None self._cache_path = CACHE_PATH @@ -333,8 +344,8 @@ class OpenStackConfig(object): :param kwargs: Additional configuration options """ - if cloud is None and 'envvars' in self._get_cloud_sections(): - cloud = 'envvars' + if cloud is None and self.envvar_key in self._get_cloud_sections(): + cloud = self.envvar_key args = self._fix_args(kwargs, argparse=argparse) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 9decf3efa..c537842b7 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -14,6 +14,7 @@ from os_client_config import cloud_config from os_client_config import config +from os_client_config import exceptions from os_client_config.tests import base @@ -36,3 +37,9 @@ class TestConfig(base.TestCase): self._assert_cloud_details(cc) cc = c.get_one_cloud('_test_cloud_no_vendor') self._assert_cloud_details(cc) + + def test_no_environ(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + self.assertRaises( + exceptions.OpenStackConfigException, c.get_one_cloud, 'envvars') diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 473c41743..363cff1a7 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -15,32 +15,36 @@ from os_client_config import cloud_config from os_client_config import config -from os_client_config import exceptions from os_client_config.tests import base import fixtures -class TestConfig(base.TestCase): +class TestEnviron(base.TestCase): - def test_get_one_cloud(self): - c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) - self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) - - def test_no_environ(self): - c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) - self.assertRaises( - exceptions.OpenStackConfigException, c.get_one_cloud, 'envvars') - - def test_environ_exists(self): + def setUp(self): + super(TestEnviron, self).setUp() self.useFixture( fixtures.EnvironmentVariable('OS_AUTH_URL', 'https://example.com')) self.useFixture( fixtures.EnvironmentVariable('OS_USERNAME', 'testuser')) self.useFixture( fixtures.EnvironmentVariable('OS_PROJECT_NAME', 'testproject')) + + def test_get_one_cloud(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) + + def test_envvar_name_override(self): + self.useFixture( + fixtures.EnvironmentVariable('OS_CLOUD_NAME', 'openstack')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('openstack') + self._assert_cloud_details(cc) + + def test_environ_exists(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud('envvars') From db39e9831ea600c8f7f02a5ddec2e0b8b5924de2 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 11 Apr 2015 09:16:56 -0400 Subject: [PATCH 050/365] Add DreamCompute to vendors list Change-Id: I4d81e221c8105d796dcd29fcd7628738486e4b00 --- os_client_config/vendors.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index eb08bbdbd..3fafe4124 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -21,6 +21,7 @@ CLOUD_DEFAULTS = dict( region_name='region-b.geo-1', dns_service_type='hpext:dns', image_api_version='1', + image_format='qcow2', ), rackspace=dict( auth=dict( @@ -29,5 +30,14 @@ CLOUD_DEFAULTS = dict( database_service_type='rax:database', compute_service_name='cloudServersOpenStack', image_api_version='2', - ) + image_format='vhd', + ), + dreamhost=dict( + auth=dict( + auth_url='https://keystone.dream.io/v2.0', + region_name='RegionOne', + ), + image_api_version='2', + image_format='raw', + ), ) From 6ec480162aac7ffe4d905a89836247077c352806 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 13 Apr 2015 10:00:58 -0400 Subject: [PATCH 051/365] Add vexxhost vexxhost has a public cloud - we should list them in our vendors file. Change-Id: Icf1276d59dadf50cabca3a7c2540121fb1cf7057 --- os_client_config/vendors.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index 3fafe4124..fec3159ba 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -40,4 +40,10 @@ CLOUD_DEFAULTS = dict( image_api_version='2', image_format='raw', ), + vexxhost=dict( + auth=dict( + auth_url='http://auth.api.thenebulacloud.com:5000/v2.0/', + region_name='ca-ymq-1', + ), + ), ) From 5bc39aea20b30df03f3ceffda697be7ece9355d3 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 13 Apr 2015 10:00:58 -0400 Subject: [PATCH 052/365] Add image information to vexxhost account Change-Id: I0e39c2a9828fb4fa73403158c3e58fb346ac9a10 --- os_client_config/vendors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index fec3159ba..96ecfcfbb 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -45,5 +45,7 @@ CLOUD_DEFAULTS = dict( auth_url='http://auth.api.thenebulacloud.com:5000/v2.0/', region_name='ca-ymq-1', ), + image_api_version='1', + image_format='qcow2', ), ) From ce6502270fb863e31c64e8e029993e9239c0d46a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 13 Apr 2015 13:04:44 -0400 Subject: [PATCH 053/365] Add runabove to vendors Change-Id: I319365aeb3a5a00498b37128c5c9fbaf018d88f4 --- os_client_config/vendors.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index 96ecfcfbb..90dccc756 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -48,4 +48,12 @@ CLOUD_DEFAULTS = dict( image_api_version='1', image_format='qcow2', ), + runabove=dict( + auth=dict( + auth_url='https://auth.runabove.io/v2.0', + ), + image_api_version='1', + image_format='qcow2', + ), + ) From 825e5b5e83385184a8472113d0d528aa770c3e95 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 13 Apr 2015 14:02:32 -0400 Subject: [PATCH 054/365] Move region_names out of auth dict region_name is not an auth parameter - this is just simply an error. Change-Id: I5cc3847932d7c51288f451b4532b71f95d8c823d --- os_client_config/vendors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index 90dccc756..70756aebf 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -35,16 +35,16 @@ CLOUD_DEFAULTS = dict( dreamhost=dict( auth=dict( auth_url='https://keystone.dream.io/v2.0', - region_name='RegionOne', ), + region_name='RegionOne', image_api_version='2', image_format='raw', ), vexxhost=dict( auth=dict( auth_url='http://auth.api.thenebulacloud.com:5000/v2.0/', - region_name='ca-ymq-1', ), + region_name='ca-ymq-1', image_api_version='1', image_format='qcow2', ), From f2e943e178e9d46ff911f224df8954a33a15a69c Mon Sep 17 00:00:00 2001 From: TerryHowe Date: Mon, 13 Apr 2015 12:45:44 -0600 Subject: [PATCH 055/365] add .venv to gitignore Change-Id: I36856a31f7a68280f9b787e5ab9ce5ac3aa0dc60 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 837459343..218bf6a45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.py[cod] +.venv # C extensions *.so From b7f38ff66144320116cb5ddf3aa39700dad42209 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 14 Apr 2015 10:26:14 -0400 Subject: [PATCH 056/365] Reset cache default to 0 None breaks the ansible inventory script. Change-Id: Iac30cdcce3a51910e0b373521263b239f7478a15 --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index a0e7672a1..8d687ef61 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -109,7 +109,7 @@ class OpenStackConfig(object): ' env vars are set') self.cloud_config['clouds'][self.envvar_key] = envvars - self._cache_max_age = None + self._cache_max_age = 0 self._cache_path = CACHE_PATH self._cache_class = 'dogpile.cache.null' self._cache_arguments = {} From b55de0e1637b1e9f6e424b070d13577af42d7fc6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 14 Apr 2015 11:23:49 -0400 Subject: [PATCH 057/365] Document vendor support information In the future, I'd like for this doc to be generated from the vendors.py file, but for now, this is great. Change-Id: Ifd0c8da5da46ba156c789f05398abcfa689f4f01 --- doc/source/index.rst | 1 + doc/source/vendor-support.rst | 86 +++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 doc/source/vendor-support.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index efde7601a..0f793a1a3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -3,6 +3,7 @@ .. toctree:: :maxdepth: 2 + vendor-support contributing installation diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst new file mode 100644 index 000000000..f74038e4b --- /dev/null +++ b/doc/source/vendor-support.rst @@ -0,0 +1,86 @@ +============== +Vendor Support +============== + +OpenStack presents deployers with many options, some of which can expose +differences to end users. `os-client-config` tries its best to collect +information about various things a user would need to know. The following +is a text representation of the vendor related defaults `os-client-config` +knows about. + +HP Cloud +-------- + +https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 + +============== ================ +Region Name Human Name +============== ================ +region-a.geo-1 US West +region-b.geo-1 US East +============== ================ + +* DNS Service Type is `hpext:dns` +* Image API Version is 1 +* Images must be in `qcow2` format + +Rackspace +--------- + +https://identity.api.rackspacecloud.com/v2.0/ + +============== ================ +Region Name Human Name +============== ================ +DFW Dallas +ORD Chicago +IAD Washington, D.C. +============== ================ + +* Database Service Type is `rax:database` +* Compute Service Name is `cloudServersOpenStack` +* Image API Version is 2 +* Images must be in `vhd` format + +Dreamhost +--------- + +https://keystone.dream.io/v2.0 + +============== ================ +Region Name Human Name +============== ================ +RegionOne Region One +============== ================ + +* Image API Version is 2 +* Images must be in `raw` format + +Vexxhost +-------- + +http://auth.api.thenebulacloud.com:5000/v2.0/ + +============== ================ +Region Name Human Name +============== ================ +ca-ymq-1 Montreal +============== ================ + +* Image API Version is 1 +* Images must be in `qcow2` format + +RunAbove +-------- + +https://auth.runabove.io/v2.0 + +============== ================ +Region Name Human Name +============== ================ +SBG-1 Strassbourg, FR +BHS-1 Beauharnois, QC +============== ================ + +* Image API Version is 1 +* Images must be in `qcow2` format From 3f8431a9de68b9e2fa24237f51d5497ca5da0295 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 30 Apr 2015 02:26:34 +0200 Subject: [PATCH 058/365] Add flag to trigger task interface for rackspace It turns out that the task interface for upload is not implied by v2 glance api. Add a boolean flag that can be consumed elsewhere. Change-Id: I2cdcfe302a73ebfa7f739399c1eeb3bc9f96ab3c --- doc/source/vendor-support.rst | 1 + os_client_config/vendors.py | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index f74038e4b..b9eb769e1 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -41,6 +41,7 @@ IAD Washington, D.C. * Compute Service Name is `cloudServersOpenStack` * Image API Version is 2 * Images must be in `vhd` format +* Images must be uploaded using the Glance Task Interface Dreamhost --------- diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index 70756aebf..f0a526609 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -30,6 +30,7 @@ CLOUD_DEFAULTS = dict( database_service_type='rax:database', compute_service_name='cloudServersOpenStack', image_api_version='2', + image_api_use_tasks=True, image_format='vhd', ), dreamhost=dict( From ace6d92a77066a04c0ff59cce5ad95ee6906d2d1 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 2 May 2015 18:16:40 -0400 Subject: [PATCH 059/365] Update vendor support to reflect v2 non-task Vexxhost and RunAbove both support v2 of glance, but do not require the task interface. Update the erroneous information. Also, there are two additional regions for Rackspace that we did not list. We don't list the Rackspace LON region because it does not work like the others and we do not know how to support it. Change-Id: Ib155d00d1d6bf7b2e5bcf4b868c268561e198611 --- doc/source/vendor-support.rst | 6 ++++-- os_client_config/vendors.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index b9eb769e1..c75f0122b 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -35,6 +35,8 @@ Region Name Human Name DFW Dallas ORD Chicago IAD Washington, D.C. +SYD Sydney +HKG Hong Kong ============== ================ * Database Service Type is `rax:database` @@ -68,7 +70,7 @@ Region Name Human Name ca-ymq-1 Montreal ============== ================ -* Image API Version is 1 +* Image API Version is 2 * Images must be in `qcow2` format RunAbove @@ -83,5 +85,5 @@ SBG-1 Strassbourg, FR BHS-1 Beauharnois, QC ============== ================ -* Image API Version is 1 +* Image API Version is 2 * Images must be in `qcow2` format diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index f0a526609..5953729da 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -46,14 +46,14 @@ CLOUD_DEFAULTS = dict( auth_url='http://auth.api.thenebulacloud.com:5000/v2.0/', ), region_name='ca-ymq-1', - image_api_version='1', + image_api_version='2', image_format='qcow2', ), runabove=dict( auth=dict( auth_url='https://auth.runabove.io/v2.0', ), - image_api_version='1', + image_api_version='2', image_format='qcow2', ), From 912af15b522423ee8f91d8b8861c4d4a090ef793 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 2 May 2015 19:27:22 -0400 Subject: [PATCH 060/365] Remove crufty lines from README Change-Id: Ibf36a67503aab5f3bcd3ba535da5f9f54ce9c28d --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index 34a1b02d8..5c3dcfaf2 100644 --- a/README.rst +++ b/README.rst @@ -169,6 +169,3 @@ Or, get all of the clouds. cloud_config = os_client_config.OpenStackConfig().get_all_clouds() for cloud in cloud_config: print(cloud.name, cloud.region, cloud.config) - -* Free software: Apache license -* Source: http://git.openstack.org/cgit/stackforge/os-client-config From dc8d21db11da1c80ad13f5626bbea63fdd649628 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 5 May 2015 14:27:40 -0600 Subject: [PATCH 061/365] Also accept .yml as a suffix The ansible world uses .yml suffixes, which brings people to try to use that suffix on clouds.yaml files. It's easy enough to accept it as a possibility. Change-Id: Iba05eab9cf4406833afafe8143794461b656b548 --- os_client_config/config.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 8d687ef61..f2ddd48b2 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -31,15 +31,22 @@ CONFIG_HOME = os.path.join(os.path.expanduser( os.environ.get('XDG_CONFIG_HOME', os.path.join('~', '.config'))), 'openstack') CONFIG_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] +YAML_SUFFIXES = ('.yaml', '.yml') CONFIG_FILES = [ - os.path.join(d, 'clouds.yaml') for d in CONFIG_SEARCH_PATH] + os.path.join(d, 'clouds' + s) + for d in CONFIG_SEARCH_PATH + for s in YAML_SUFFIXES +] CACHE_PATH = os.path.join(os.path.expanduser( os.environ.get('XDG_CACHE_PATH', os.path.join('~', '.cache'))), 'openstack') BOOL_KEYS = ('insecure', 'cache') VENDOR_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] VENDOR_FILES = [ - os.path.join(d, 'clouds-public.yaml') for d in VENDOR_SEARCH_PATH] + os.path.join(d, 'clouds-public' + s) + for d in VENDOR_SEARCH_PATH + for s in YAML_SUFFIXES +] def set_default(key, value): @@ -126,14 +133,13 @@ class OpenStackConfig(object): 'arguments', self._cache_arguments) def _load_config_file(self): - for path in self._config_files: - if os.path.exists(path): - with open(path, 'r') as f: - return yaml.safe_load(f) - return dict(clouds=dict()) + return self._load_yaml_file(self._config_files) def _load_vendor_file(self): - for path in self._vendor_files: + return self._load_yaml_file(self._vendor_files) + + def _load_yaml_file(self, filelist): + for path in filelist: if os.path.exists(path): with open(path, 'r') as f: return yaml.safe_load(f) From f6d08765cd8ceb071e99453d3f17faabb5878431 Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Thu, 7 May 2015 17:58:54 +0000 Subject: [PATCH 062/365] get_one_cloud should use auth merge Currently, get_one_cloud does not merge the auth dictionary, which means passing any auth value overrides all auth: values in clouds.yaml. Change-Id: I22c33648e32cc7ce8fc163433b7c72912c28beb9 --- os_client_config/config.py | 5 ++++- os_client_config/tests/test_config.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 8d687ef61..b05fc44f3 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -357,7 +357,10 @@ class OpenStackConfig(object): # Can't just do update, because None values take over for (key, val) in iter(args.items()): if val is not None: - config[key] = val + if key == 'auth' and config[key] is not None: + config[key] = _auth_update(config[key], val) + else: + config[key] = val for key in BOOL_KEYS: if key in config: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index c537842b7..274b188c1 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -43,3 +43,9 @@ class TestConfig(base.TestCase): vendor_files=[self.vendor_yaml]) self.assertRaises( exceptions.OpenStackConfigException, c.get_one_cloud, 'envvars') + + def test_get_one_cloud_auth_merge(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml]) + cc = c.get_one_cloud(cloud='_test_cloud_', auth={'username': 'user'}) + self.assertEqual('user', cc.auth['username']) + self.assertEqual('testpass', cc.auth['password']) From 33634dda7219f334c5545cb93673311c8bc99130 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 7 May 2015 18:50:28 -0400 Subject: [PATCH 063/365] Add flag to indicate where floating ips come from One of the things that's not possible to discover is where floating ips come from. You'd think that looking for a neutron endpoint would do it, but you'd be oh-so-wrong. Change-Id: I10a0c6f37afb409af0078cede3eac2eaa0ff4f04 --- doc/source/vendor-support.rst | 5 +++++ os_client_config/defaults.py | 2 ++ os_client_config/vendors.py | 3 +++ 3 files changed, 10 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index c75f0122b..1d5aff35f 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -23,6 +23,7 @@ region-b.geo-1 US East * DNS Service Type is `hpext:dns` * Image API Version is 1 * Images must be in `qcow2` format +* Floating IPs are provided by Neutron Rackspace --------- @@ -44,6 +45,7 @@ HKG Hong Kong * Image API Version is 2 * Images must be in `vhd` format * Images must be uploaded using the Glance Task Interface +* Floating IPs are not needed Dreamhost --------- @@ -58,6 +60,7 @@ RegionOne Region One * Image API Version is 2 * Images must be in `raw` format +* Floating IPs are provided by Neutron Vexxhost -------- @@ -72,6 +75,7 @@ ca-ymq-1 Montreal * Image API Version is 2 * Images must be in `qcow2` format +* Floating IPs are not needed RunAbove -------- @@ -87,3 +91,4 @@ BHS-1 Beauharnois, QC * Image API Version is 2 * Images must be in `qcow2` format +* Floating IPs are not needed diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index 5f70d6ee4..905950201 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -16,8 +16,10 @@ _defaults = dict( auth_type='password', compute_api_version='2', identity_api_version='2', + image_api_use_tasks=False, image_api_version='1', network_api_version='2', object_api_version='1', volume_api_version='1', + floating_ip_source='neutron', ) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index 5953729da..2eecfa38e 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -32,6 +32,7 @@ CLOUD_DEFAULTS = dict( image_api_version='2', image_api_use_tasks=True, image_format='vhd', + floating_ip_source=None, ), dreamhost=dict( auth=dict( @@ -48,6 +49,7 @@ CLOUD_DEFAULTS = dict( region_name='ca-ymq-1', image_api_version='2', image_format='qcow2', + floating_ip_source=None, ), runabove=dict( auth=dict( @@ -55,6 +57,7 @@ CLOUD_DEFAULTS = dict( ), image_api_version='2', image_format='qcow2', + floating_ip_source=None, ), ) From 2f1e6c13bee31a7d43ef80847d764648afdb9578 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 May 2015 08:49:59 -0400 Subject: [PATCH 064/365] Only add fall through cloud as a fall through We only want do define the 'openstack' cloud if there are neither environment varaibles nor config files. We need to define it as a place to put passed-in-parameters in the case of neither, but we don't want it any other time. The behavior can now be described as: - If you have a config file, you will get the clouds listed in it - If you have environment variable, you will get a cloud named 'envvars' - If you have neither, you will get a cloud named 'defaults' Change-Id: I6752c1ccecf1ef979b2603246eeaab7da360c8a4 --- README.rst | 12 +++++--- os_client_config/config.py | 39 ++++++++++++++------------ os_client_config/tests/base.py | 4 +++ os_client_config/tests/test_config.py | 12 ++++++++ os_client_config/tests/test_environ.py | 15 +++++++--- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index 5c3dcfaf2..2c27153ec 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,10 @@ put in a config file. It will read environment variables and config files, and it also contains some vendor specific default values so that you don't have to know extra info to use OpenStack +* If you have a config file, you will get the clouds listed in it +* If you have environment variables, you will get a cloud named 'envvars' +* If you have neither, you will get a cloud named 'defaults' with base defaults + Environment Variables --------------------- @@ -16,10 +20,10 @@ os-client-config honors all of the normal `OS_*` variables. It does not provide backwards compatibility to service-specific variables such as `NOVA_USERNAME`. -If you have OpenStack environment variables seet and no config files, -os-client-config will produce a cloud config object named "envvars" containing -your values from the environment. If you don't like the name "envvars", that's -ok, you can override it by setting `OS_CLOUD_NAME`. +If you have OpenStack environment variables set, os-client-config will produce +a cloud config object named "envvars" containing your values from the +environment. If you don't like the name "envvars", that's ok, you can override +it by setting `OS_CLOUD_NAME`. Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type diff --git a/os_client_config/config.py b/os_client_config/config.py index e926555bc..45770d97b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -91,31 +91,34 @@ class OpenStackConfig(object): self.defaults = dict(defaults._defaults) - # use a config file if it exists where expected + # First, use a config file if it exists where expected self.cloud_config = self._load_config_file() - if not self.cloud_config: - self.cloud_config = dict( - clouds=dict(openstack=dict(self.defaults))) - self.envvar_key = os.environ.pop('OS_CLOUD_NAME', None) - if self.envvar_key: - if self.envvar_key in self.cloud_config['clouds']: - raise exceptions.OpenStackConfigException( - 'clouds.yaml defines a cloud named "{0}", but' - ' OS_CLOUD_NAME is also set to "{0}". Please rename' - ' either your environment based cloud, or one of your' - ' file-based clouds.'.format(self.envvar_key)) - else: - self.envvar_key = 'envvars' + if not self.cloud_config: + self.cloud_config = {'clouds': {}} + if 'clouds' not in self.cloud_config: + self.cloud_config['clouds'] = {} + + # Next, process environment variables and add them to the mix + self.envvar_key = os.environ.pop('OS_CLOUD_NAME', 'envvars') + if self.envvar_key in self.cloud_config['clouds']: + raise exceptions.OpenStackConfigException( + 'clouds.yaml defines a cloud named "{0}", but' + ' OS_CLOUD_NAME is also set to "{0}". Please rename' + ' either your environment based cloud, or one of your' + ' file-based clouds.'.format(self.envvar_key)) envvars = _get_os_environ() if envvars: - if self.envvar_key in self.cloud_config['clouds']: - raise exceptions.OpenStackConfigException( - 'clouds.yaml defines a cloud named {0}, and OS_*' - ' env vars are set') self.cloud_config['clouds'][self.envvar_key] = envvars + # Finally, fall through and make a cloud that starts with defaults + # because we need somewhere to put arguments, and there are neither + # config files or env vars + if not self.cloud_config['clouds']: + self.cloud_config = dict( + clouds=dict(defaults=dict(self.defaults))) + self._cache_max_age = 0 self._cache_path = CACHE_PATH self._cache_class = 'dogpile.cache.null' diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index f6e815d9c..1169051f8 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -59,6 +59,9 @@ USER_CONF = { }, 'cache': {'max_age': 1}, } +NO_CONF = { + 'cache': {'max_age': 1}, +} def _write_yaml(obj): @@ -80,6 +83,7 @@ class TestCase(base.BaseTestCase): conf['cache']['path'] = tdir.path self.cloud_yaml = _write_yaml(conf) self.vendor_yaml = _write_yaml(VENDOR_CONF) + self.no_yaml = _write_yaml(NO_CONF) # Isolate the test runs from the environment # Do this as two loops because you can't modify the dict in a loop diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 274b188c1..ae573a638 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -12,6 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +import os + +import fixtures + from os_client_config import cloud_config from os_client_config import config from os_client_config import exceptions @@ -44,6 +48,14 @@ class TestConfig(base.TestCase): self.assertRaises( exceptions.OpenStackConfigException, c.get_one_cloud, 'envvars') + def test_fallthrough(self): + c = config.OpenStackConfig(config_files=[self.no_yaml], + vendor_files=[self.no_yaml]) + for k in os.environ.keys(): + if k.startswith('OS_'): + self.useFixture(fixtures.EnvironmentVariable(k)) + c.get_one_cloud(cloud='defaults') + def test_get_one_cloud_auth_merge(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml]) cc = c.get_one_cloud(cloud='_test_cloud_', auth={'username': 'user'}) diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 363cff1a7..e60c73b31 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -15,6 +15,7 @@ from os_client_config import cloud_config from os_client_config import config +from os_client_config import exceptions from os_client_config.tests import base import fixtures @@ -36,12 +37,18 @@ class TestEnviron(base.TestCase): vendor_files=[self.vendor_yaml]) self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) - def test_envvar_name_override(self): - self.useFixture( - fixtures.EnvironmentVariable('OS_CLOUD_NAME', 'openstack')) + def test_no_fallthrough(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud('openstack') + self.assertRaises( + exceptions.OpenStackConfigException, c.get_one_cloud, 'openstack') + + def test_envvar_name_override(self): + self.useFixture( + fixtures.EnvironmentVariable('OS_CLOUD_NAME', 'override')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('override') self._assert_cloud_details(cc) def test_environ_exists(self): From b9cdb7666580d81b956a045c0d10d697f0963164 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 May 2015 09:41:36 -0400 Subject: [PATCH 065/365] Sort defaults list for less conflicts Change-Id: Ic27c50f745a093cc20e3f22f09698f7ae643bc83 --- os_client_config/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index 905950201..bf9a26dcf 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -15,11 +15,11 @@ _defaults = dict( auth_type='password', compute_api_version='2', + floating_ip_source='neutron', identity_api_version='2', image_api_use_tasks=False, image_api_version='1', network_api_version='2', object_api_version='1', volume_api_version='1', - floating_ip_source='neutron', ) From b16c49cfd6a0ee659e4493ef959e0483e93d350a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 May 2015 09:50:33 -0400 Subject: [PATCH 066/365] Add default versions for trove and ironic Change-Id: Ib7af38664cfbe75c78c70693117f1193c4beb7e6 --- os_client_config/defaults.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index bf9a26dcf..f0476a60b 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -14,7 +14,9 @@ _defaults = dict( auth_type='password', + baremetal_api_version='1', compute_api_version='2', + database_api_version='1.0', floating_ip_source='neutron', identity_api_version='2', image_api_use_tasks=False, From 508c240f851dbfd5aa1d1223910ecb1df6a33b2a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 May 2015 09:54:01 -0400 Subject: [PATCH 067/365] Expose function to get defaults dict shade needs sane defaults for some of these too, but it's entirely legitimate to use shade without having first created an os_client_config object. Adding a function to get a copy of the defaults dict allows it to be consumed as a single source or truth for default values. Change-Id: I616d9492c7e0f53c48519cc8dacf3dfbd0082e36 --- os_client_config/config.py | 4 ++-- os_client_config/defaults.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 45770d97b..64afdbdd3 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -60,7 +60,7 @@ def get_boolean(value): def _get_os_environ(): - ret = dict(defaults._defaults) + ret = defaults.get_defaults() environkeys = [k for k in os.environ.keys() if k.startswith('OS_')] if not environkeys: return None @@ -89,7 +89,7 @@ class OpenStackConfig(object): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES - self.defaults = dict(defaults._defaults) + self.defaults = defaults.get_defaults() # First, use a config file if it exists where expected self.cloud_config = self._load_config_file() diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index f0476a60b..14b820949 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -25,3 +25,7 @@ _defaults = dict( object_api_version='1', volume_api_version='1', ) + + +def get_defaults(): + return _defaults.copy() From 9b9e3d0d329b541960c8e28f897718a62a74ddf1 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 May 2015 21:10:13 -0400 Subject: [PATCH 068/365] Add UnitedStack With Keystone v3 even! Change-Id: If9445d99ccfa5f15ca3760bee4da900f302dc698 --- doc/source/vendor-support.rst | 17 +++++++++++++++++ os_client_config/vendors.py | 10 +++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 1d5aff35f..dea27d13d 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -92,3 +92,20 @@ BHS-1 Beauharnois, QC * Image API Version is 2 * Images must be in `qcow2` format * Floating IPs are not needed + +UnitedStack +----------- + +https://identity.api.ustack.com/v3 + +============== ================ +Region Name Human Name +============== ================ +bj1 Beijing +gd1 Guangdong +============== ================ + +* Identity API Version is 3 +* Image API Version is 2 +* Images must be in `raw` format +* Floating IPs are not needed diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index 2eecfa38e..dec63dc2e 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -59,5 +59,13 @@ CLOUD_DEFAULTS = dict( image_format='qcow2', floating_ip_source=None, ), - + unitedstack=dict( + auth=dict( + auth_url='https://identity.api.ustack.com/v3', + ), + identity_api_version='3', + image_api_version='2', + image_format='raw', + floating_ip_source=None, + ), ) From 4b40133e2199e11ccd5dc48c2ad60ac06d056d0a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 11 May 2015 16:24:28 -0400 Subject: [PATCH 069/365] Use appdirs for platform-independent locations Cache, data and config files live rooted in different places across different OS's. Use appdirs to find where. Depends-On: Ic939dea11b7476ec504d2bf65854a0781b1bfb39 Change-Id: I7338ae1d0442e0c5cc1ec4ae4d619fac319a4a28 --- README.rst | 22 ++++++++++++++++++++++ os_client_config/config.py | 29 +++++++++++++++++++---------- requirements.txt | 1 + 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 2c27153ec..4389b680b 100644 --- a/README.rst +++ b/README.rst @@ -50,6 +50,28 @@ Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type for trove (because you're using Rackspace) set: +Site Specific File Locations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In addition to `~/.config/openstack` and `/etc/openstack` - some platforms +have other locations they like to put things. `os-client-config` will also +look in an OS specific config dir + +* `USER_CONFIG_DIR` +* `SITE_CONFIG_DIR` + +`USER_CONFIG_DIR` is different on Linux, OSX and Windows. + +* Linux: `~/.config/openstack` +* OSX: `~/Library/Application Support/openstack` +* Windows: `C:\\Users\\USERNAME\\AppData\\Local\\OpenStack\\openstack` + +`SITE_CONFIG_DIR` is different on Linux, OSX and Windows. + +* Linux: `/etc/openstack` +* OSX: `/Library/Application Support/openstack` +* Windows: `C:\\ProgramData\\OpenStack\\openstack` + :: database_service_type: 'rax:database' diff --git a/os_client_config/config.py b/os_client_config/config.py index 64afdbdd3..dd4f37f2b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -15,6 +15,7 @@ import os +import appdirs import yaml try: @@ -27,27 +28,35 @@ from os_client_config import defaults from os_client_config import exceptions from os_client_config import vendors -CONFIG_HOME = os.path.join(os.path.expanduser( - os.environ.get('XDG_CONFIG_HOME', os.path.join('~', '.config'))), - 'openstack') -CONFIG_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] +APPDIRS = appdirs.AppDirs('openstack', 'OpenStack', multipath='/etc') +CONFIG_HOME = APPDIRS.user_config_dir +CACHE_PATH = APPDIRS.user_cache_dir + +UNIX_CONFIG_HOME = os.path.join( + os.path.expanduser(os.path.join('~', '.config')), 'openstack') +UNIX_SITE_CONFIG_HOME = '/etc/openstack' + +SITE_CONFIG_HOME = APPDIRS.site_config_dir + +CONFIG_SEARCH_PATH = [ + os.getcwd(), + CONFIG_HOME, UNIX_CONFIG_HOME, + SITE_CONFIG_HOME, UNIX_SITE_CONFIG_HOME +] YAML_SUFFIXES = ('.yaml', '.yml') CONFIG_FILES = [ os.path.join(d, 'clouds' + s) for d in CONFIG_SEARCH_PATH for s in YAML_SUFFIXES ] -CACHE_PATH = os.path.join(os.path.expanduser( - os.environ.get('XDG_CACHE_PATH', os.path.join('~', '.cache'))), - 'openstack') -BOOL_KEYS = ('insecure', 'cache') -VENDOR_SEARCH_PATH = [os.getcwd(), CONFIG_HOME, '/etc/openstack'] VENDOR_FILES = [ os.path.join(d, 'clouds-public' + s) - for d in VENDOR_SEARCH_PATH + for d in CONFIG_SEARCH_PATH for s in YAML_SUFFIXES ] +BOOL_KEYS = ('insecure', 'cache') + def set_default(key, value): defaults._defaults[key] = value diff --git a/requirements.txt b/requirements.txt index 498c5c336..894a70acc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. PyYAML>=3.1.0 +appdirs>=1.3.0 From 948aa01d0c15c1728deb954b2764682705176c9c Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Mon, 11 May 2015 17:29:42 -0500 Subject: [PATCH 070/365] Add tests for OSC usage OSC uses os-client-config as an intermediate handler in the argparse-to-final-options sequence. * Validate the expected usage of the argparse arguement to get_one_cloud(). This turned out to be a problem with the way set_defaults() was implemented and has been withdrawn until set_defaults is re-worked as a kwarg to OpenStackCloud.__init__() * Validate setting the default auth_type value Change-Id: Idae91962f05d787cecf4a59fac01e9321bc69687 --- os_client_config/tests/test_config.py | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index ae573a638..f4bf6dbbe 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import argparse import os import fixtures @@ -61,3 +62,52 @@ class TestConfig(base.TestCase): cc = c.get_one_cloud(cloud='_test_cloud_', auth={'username': 'user'}) self.assertEqual('user', cc.auth['username']) self.assertEqual('testpass', cc.auth['password']) + + +class TestConfigArgparse(base.TestCase): + + def setUp(self): + super(TestConfigArgparse, self).setUp() + + self.options = argparse.Namespace( + region_name='other-test-region', + snack_type='cookie', + ) + + def test_get_one_cloud_argparse(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud(cloud='_test_cloud_', argparse=self.options) + self._assert_cloud_details(cc) + self.assertEqual(cc.region_name, 'other-test-region') + self.assertEqual(cc.snack_type, 'cookie') + + def test_get_one_cloud_just_argparse(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud(cloud='', argparse=self.options) + self.assertIsNone(cc.cloud) + self.assertNotIn('username', cc.auth) + self.assertEqual(cc.region_name, 'other-test-region') + self.assertEqual(cc.snack_type, 'cookie') + + def test_get_one_cloud_no_argparse(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud(cloud='_test_cloud_', argparse=None) + self._assert_cloud_details(cc) + self.assertEqual(cc.region_name, 'test-region') + self.assertIsNone(cc.snack_type) + + +class TestConfigDefault(base.TestCase): + + def test_set_no_default(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud(cloud='_test_cloud_', argparse=None) + self._assert_cloud_details(cc) + self.assertEqual(cc.auth_type, 'password') From cbdc7c70801d7a0bd9a6ac51e0b9b6f4dc9e7b8d Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Tue, 12 May 2015 14:26:27 -0500 Subject: [PATCH 071/365] Change overriding defaults to kwarg Forcibly changing the global defaults isn't the nicest thing to do and turns out to be a pain during tests. Make overriding defaults into an arg to OpenStackConfig.__init__(). OSC is the only known user of set_default(), once this is released and in global-requirements OSC an be updated then set_default() can be removed. Change-Id: Iddbc66398e89f06f1d665d7b0ef243bc786c8e36 --- os_client_config/config.py | 9 ++++++++- os_client_config/tests/test_config.py | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 64afdbdd3..5b6439550 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -49,6 +49,10 @@ VENDOR_FILES = [ ] +# NOTE(dtroyer): This turns out to be not the best idea so let's move +# overriding defaults to a kwarg to OpenStackConfig.__init__() +# Remove this sometime in June 2015 once OSC is comfortably +# changed-over and global-defaults is updated. def set_default(key, value): defaults._defaults[key] = value @@ -85,11 +89,14 @@ def _auth_update(old_dict, new_dict): class OpenStackConfig(object): - def __init__(self, config_files=None, vendor_files=None): + def __init__(self, config_files=None, vendor_files=None, + override_defaults=None): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES self.defaults = defaults.get_defaults() + if override_defaults: + self.defaults.update(override_defaults) # First, use a config file if it exists where expected self.cloud_config = self._load_config_file() diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index ae573a638..8df62d60a 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -18,6 +18,7 @@ import fixtures from os_client_config import cloud_config from os_client_config import config +from os_client_config import defaults from os_client_config import exceptions from os_client_config.tests import base @@ -29,6 +30,31 @@ class TestConfig(base.TestCase): vendor_files=[self.vendor_yaml]) self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) + def test_get_one_cloud_auth_defaults(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml]) + cc = c.get_one_cloud(cloud='_test_cloud_', auth={'username': 'user'}) + self.assertEqual('user', cc.auth['username']) + self.assertEqual( + defaults._defaults['auth_type'], + cc.auth_type, + ) + self.assertEqual( + defaults._defaults['identity_api_version'], + cc.identity_api_version, + ) + + def test_get_one_cloud_auth_override_defaults(self): + default_options = {'auth_type': 'token'} + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + override_defaults=default_options) + cc = c.get_one_cloud(cloud='_test_cloud_', auth={'username': 'user'}) + self.assertEqual('user', cc.auth['username']) + self.assertEqual('token', cc.auth_type) + self.assertEqual( + defaults._defaults['identity_api_version'], + cc.identity_api_version, + ) + def test_get_one_cloud_with_config_files(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From 434d51aac292609162f738eea4611e2a52df8a0f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 26 May 2015 14:49:56 -0400 Subject: [PATCH 072/365] Don't pass None as the cloud name cloud names want to be strings. If there is no cloud name, stringify None. Change-Id: I4fa5bb8ac03baf464cfa00b451627f63ff847fdf --- os_client_config/config.py | 6 +++++- os_client_config/tests/test_config.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 88e4b4c2e..eb7b5c07b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -404,8 +404,12 @@ class OpenStackConfig(object): if hasattr(value, 'format'): config[key] = value.format(**config) + if cloud is None: + cloud_name = '' + else: + cloud_name = str(cloud) return cloud_config.CloudConfig( - name=cloud, region=config['region_name'], config=config) + name=cloud_name, region=config['region_name'], config=config) if __name__ == '__main__': config = OpenStackConfig().get_all_clouds() diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index d196a8691..bb815293b 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -29,7 +29,9 @@ class TestConfig(base.TestCase): def test_get_one_cloud(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - self.assertIsInstance(c.get_one_cloud(), cloud_config.CloudConfig) + cloud = c.get_one_cloud() + self.assertIsInstance(cloud, cloud_config.CloudConfig) + self.assertEqual(cloud.name, '') def test_get_one_cloud_auth_defaults(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml]) From b51f9f841651710ed088867b9047a7311e39cdea Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 27 May 2015 08:01:40 -0400 Subject: [PATCH 073/365] Rename cloud to profile The cloud parameter was confusing people, especially since it was inside of a dict that was named "clouds". profile was suggested as less confusing, which seems fine. Continue processing cloud as a parameter so that we don't break anyway, but change the docs to only mention profile. Change-Id: Idf3d089703985ecc60f23a3c780ddcab914aa678 --- README.rst | 16 ++++++++-------- os_client_config/config.py | 13 +++++++------ os_client_config/tests/base.py | 4 ++-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 4389b680b..193f0a112 100644 --- a/README.rst +++ b/README.rst @@ -82,7 +82,7 @@ An example config file is probably helpful: clouds: mordred: - cloud: hp + profile: hp auth: username: mordred@inaugust.com password: XXXXXXXXX @@ -99,7 +99,7 @@ An example config file is probably helpful: region_name: region-b.geo-1 dns_service_type: hpext:dns infra: - cloud: rackspace + profile: rackspace auth: username: openstackci password: XXXXXXXX @@ -107,11 +107,11 @@ An example config file is probably helpful: region_name: DFW,ORD,IAD You may note a few things. First, since auth_url settings are silly -and embarrasingly ugly, known cloud vendors are included and may be referrenced -by name. One of the benefits of that is that auth_url isn't the only thing -the vendor defaults contain. For instance, since Rackspace lists -`rax:database` as the service type for trove, os-client-config knows that -so that you don't have to. +and embarrasingly ugly, known cloud vendor profile information is included and +may be referrenced by name. One of the benefits of that is that auth_url +isn't the only thing the vendor defaults contain. For instance, since +Rackspace lists `rax:database` as the service type for trove, os-client-config +knows that so that you don't have to. Also, region_name can be a list of regions. When you call get_all_clouds, you'll get a cloud config object for each cloud/region combo. @@ -159,7 +159,7 @@ are connecting to OpenStack can share a cache should you desire. - 127.0.0.1 clouds: mordred: - cloud: hp + profile: hp auth: username: mordred@inaugust.com password: XXXXXXXXX diff --git a/os_client_config/config.py b/os_client_config/config.py index eb7b5c07b..8fc1644cf 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -202,15 +202,16 @@ class OpenStackConfig(object): # Get the defaults cloud.update(self.defaults) - # yes, I know the next line looks silly - if 'cloud' in our_cloud: - cloud_name = our_cloud['cloud'] + # Expand a profile if it exists. 'cloud' is an old confusing name + # for this. + profile_name = our_cloud.get('profile', our_cloud.get('cloud', None)) + if profile_name: vendor_file = self._load_vendor_file() - if vendor_file and cloud_name in vendor_file['public-clouds']: - _auth_update(cloud, vendor_file['public-clouds'][cloud_name]) + if vendor_file and profile_name in vendor_file['public-clouds']: + _auth_update(cloud, vendor_file['public-clouds'][profile_name]) else: try: - _auth_update(cloud, vendors.CLOUD_DEFAULTS[cloud_name]) + _auth_update(cloud, vendors.CLOUD_DEFAULTS[profile_name]) except KeyError: # Can't find the requested vendor config, go about business pass diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 1169051f8..b6834bf8c 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -40,7 +40,7 @@ VENDOR_CONF = { USER_CONF = { 'clouds': { '_test_cloud_': { - 'cloud': '_test_cloud_in_our_cloud', + 'profile': '_test_cloud_in_our_cloud', 'auth': { 'username': 'testuser', 'password': 'testpass', @@ -48,7 +48,7 @@ USER_CONF = { 'region_name': 'test-region', }, '_test_cloud_no_vendor': { - 'cloud': '_test_non_existant_cloud', + 'profile': '_test_non_existant_cloud', 'auth': { 'username': 'testuser', 'password': 'testpass', From a5dd46af2ede29580450ffad46ad1888cca4a9ac Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 27 May 2015 08:04:39 -0400 Subject: [PATCH 074/365] Expose method for getting a list of cloud names Knowing what cloud names os-client-config found can be useful for introspection or investigation. Change-Id: I77ab133634236d2a7a59ea805a6d650854165030 --- os_client_config/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 8fc1644cf..813d0a139 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -185,7 +185,7 @@ class OpenStackConfig(object): def _get_region(self, cloud=None): return self._get_regions(cloud).split(',')[0] - def _get_cloud_sections(self): + def get_cloud_names(self): return self.cloud_config['clouds'].keys() def _get_base_cloud_config(self, name): @@ -267,7 +267,7 @@ class OpenStackConfig(object): clouds = [] - for cloud in self._get_cloud_sections(): + for cloud in self.get_cloud_names(): for region in self._get_regions(cloud).split(','): clouds.append(self.get_one_cloud(cloud, region_name=region)) return clouds @@ -370,7 +370,7 @@ class OpenStackConfig(object): :param kwargs: Additional configuration options """ - if cloud is None and self.envvar_key in self._get_cloud_sections(): + if cloud is None and self.envvar_key in self.get_cloud_names(): cloud = self.envvar_key args = self._fix_args(kwargs, argparse=argparse) From f3eb3d47bc51c5c4dbf8a609c27e99cf9abaca53 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 27 May 2015 08:14:47 -0400 Subject: [PATCH 075/365] Normalize all keys down to _ instead of - It's really common to use the config dict in a **kwargs context. Therefore, normalize every key with a - to use _ instead. Testing for this is done by changing one of the base region_name args to be region-name. Change-Id: Ibd834f7a70cf2285619b5499492858b21635ba57 --- os_client_config/config.py | 16 +++++++++++++--- os_client_config/tests/base.py | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 813d0a139..c5b6d9bb3 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -161,7 +161,17 @@ class OpenStackConfig(object): for path in filelist: if os.path.exists(path): with open(path, 'r') as f: - return yaml.safe_load(f) + return self._normalize_keys(yaml.safe_load(f)) + + def _normalize_keys(self, config): + new_config = {} + for key, value in config.items(): + key = key.replace('-', '_') + if isinstance(value, dict): + new_config[key] = self._normalize_keys(value) + else: + new_config[key] = value + return new_config def get_cache_max_age(self): return self._cache_max_age @@ -207,8 +217,8 @@ class OpenStackConfig(object): profile_name = our_cloud.get('profile', our_cloud.get('cloud', None)) if profile_name: vendor_file = self._load_vendor_file() - if vendor_file and profile_name in vendor_file['public-clouds']: - _auth_update(cloud, vendor_file['public-clouds'][profile_name]) + if vendor_file and profile_name in vendor_file['public_clouds']: + _auth_update(cloud, vendor_file['public_clouds'][profile_name]) else: try: _auth_update(cloud, vendors.CLOUD_DEFAULTS[profile_name]) diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index b6834bf8c..81b499556 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -54,7 +54,7 @@ USER_CONF = { 'password': 'testpass', 'project_name': 'testproject', }, - 'region_name': 'test-region', + 'region-name': 'test-region', }, }, 'cache': {'max_age': 1}, From 5a4f809caf2223121803debab2e6ac52bc943116 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 27 May 2015 08:46:21 -0400 Subject: [PATCH 076/365] Capture the filename used for config Consumers who may want to watch for config file changes would find such a thing interesting information. Change-Id: I66c59c6acdbb930f013d4742d5b3cc7e35a922d4 --- os_client_config/config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index c5b6d9bb3..b9c89c296 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -108,7 +108,7 @@ class OpenStackConfig(object): self.defaults.update(override_defaults) # First, use a config file if it exists where expected - self.cloud_config = self._load_config_file() + self.config_filename, self.cloud_config = self._load_config_file() if not self.cloud_config: self.cloud_config = {'clouds': {}} @@ -161,7 +161,8 @@ class OpenStackConfig(object): for path in filelist: if os.path.exists(path): with open(path, 'r') as f: - return self._normalize_keys(yaml.safe_load(f)) + return path, self._normalize_keys(yaml.safe_load(f)) + return (None, None) def _normalize_keys(self, config): new_config = {} @@ -216,7 +217,7 @@ class OpenStackConfig(object): # for this. profile_name = our_cloud.get('profile', our_cloud.get('cloud', None)) if profile_name: - vendor_file = self._load_vendor_file() + vendor_filename, vendor_file = self._load_vendor_file() if vendor_file and profile_name in vendor_file['public_clouds']: _auth_update(cloud, vendor_file['public_clouds'][profile_name]) else: From 5c60aad725b0b98008ee467c5130931339c12d48 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 27 May 2015 08:50:32 -0400 Subject: [PATCH 077/365] Add an equality method for CloudConfig In order to track if a config has changed, we need to be able to compare the CloudConfig objects for equality. Change-Id: Icdd9acede81bc5fba60d877194048e24a62c9e5d --- os_client_config/cloud_config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index c17bfc2b7..d893b0b5f 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -32,3 +32,7 @@ class CloudConfig(object): def __iter__(self): return self.config.__iter__() + + def __eq__(self, other): + return (self.name == other.name and self.region == other.region + and self.config == other.config) From 9b98ee0e098cc33eb976a9d5e917d10a0ad7968c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 27 May 2015 15:52:54 -0400 Subject: [PATCH 078/365] Add inequality method One method does not imply the other. Change-Id: Ifcd39ee188d88d9490f67b5644a941d4d1c6ec38 --- os_client_config/cloud_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index d893b0b5f..347eb7c48 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -36,3 +36,6 @@ class CloudConfig(object): def __eq__(self, other): return (self.name == other.name and self.region == other.region and self.config == other.config) + + def __ne__(self, other): + return not self == other From 2580c0aa0112d94606879c8f8864e34602ddaab8 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Wed, 27 May 2015 16:09:49 -0400 Subject: [PATCH 079/365] Add tests for cloud config comparison Unit tests are good. Change-Id: Iaad73898daf6b00839be5a134558d53b95f0dd65 --- os_client_config/tests/test_cloud_config.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 36386e50e..1f79b3e71 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -42,3 +42,20 @@ class TestCloudConfig(base.TestCase): cc = cloud_config.CloudConfig("test1", "region-al", fake_config_dict) self.assertTrue('a' in cc) self.assertFalse('x' in cc) + + def test_equality(self): + cc1 = cloud_config.CloudConfig("test1", "region-al", fake_config_dict) + cc2 = cloud_config.CloudConfig("test1", "region-al", fake_config_dict) + self.assertEqual(cc1, cc2) + + def test_inequality(self): + cc1 = cloud_config.CloudConfig("test1", "region-al", fake_config_dict) + + cc2 = cloud_config.CloudConfig("test2", "region-al", fake_config_dict) + self.assertNotEqual(cc1, cc2) + + cc2 = cloud_config.CloudConfig("test1", "region-xx", fake_config_dict) + self.assertNotEqual(cc1, cc2) + + cc2 = cloud_config.CloudConfig("test1", "region-al", {}) + self.assertNotEqual(cc1, cc2) From 88c5e08599d70bde80abea45404ead57d5a17388 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 27 May 2015 22:44:08 -0400 Subject: [PATCH 080/365] Don't normalize too deeply We were normalizing the entire clouds.yaml output - but the keys underneath clouds: are names of clouds. We should not touch them. Instead - normalize the keys of the config dict we return. Change-Id: I792475d701d31201a9be6d54e066227d97b7ce5f --- os_client_config/config.py | 9 +++++---- os_client_config/tests/base.py | 2 +- os_client_config/tests/test_config.py | 14 +++++++------- os_client_config/tests/test_environ.py | 4 ++-- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index b9c89c296..7b52912ed 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -161,7 +161,7 @@ class OpenStackConfig(object): for path in filelist: if os.path.exists(path): with open(path, 'r') as f: - return path, self._normalize_keys(yaml.safe_load(f)) + return path, yaml.safe_load(f) return (None, None) def _normalize_keys(self, config): @@ -218,8 +218,8 @@ class OpenStackConfig(object): profile_name = our_cloud.get('profile', our_cloud.get('cloud', None)) if profile_name: vendor_filename, vendor_file = self._load_vendor_file() - if vendor_file and profile_name in vendor_file['public_clouds']: - _auth_update(cloud, vendor_file['public_clouds'][profile_name]) + if vendor_file and profile_name in vendor_file['public-clouds']: + _auth_update(cloud, vendor_file['public-clouds'][profile_name]) else: try: _auth_update(cloud, vendors.CLOUD_DEFAULTS[profile_name]) @@ -421,7 +421,8 @@ class OpenStackConfig(object): else: cloud_name = str(cloud) return cloud_config.CloudConfig( - name=cloud_name, region=config['region_name'], config=config) + name=cloud_name, region=config['region_name'], + config=self._normalize_keys(config)) if __name__ == '__main__': config = OpenStackConfig().get_all_clouds() diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 81b499556..6a412bfd6 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -39,7 +39,7 @@ VENDOR_CONF = { } USER_CONF = { 'clouds': { - '_test_cloud_': { + '_test-cloud_': { 'profile': '_test_cloud_in_our_cloud', 'auth': { 'username': 'testuser', diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index bb815293b..5a7fc1a94 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -35,7 +35,7 @@ class TestConfig(base.TestCase): def test_get_one_cloud_auth_defaults(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml]) - cc = c.get_one_cloud(cloud='_test_cloud_', auth={'username': 'user'}) + cc = c.get_one_cloud(cloud='_test-cloud_', auth={'username': 'user'}) self.assertEqual('user', cc.auth['username']) self.assertEqual( defaults._defaults['auth_type'], @@ -50,7 +50,7 @@ class TestConfig(base.TestCase): default_options = {'auth_type': 'token'} c = config.OpenStackConfig(config_files=[self.cloud_yaml], override_defaults=default_options) - cc = c.get_one_cloud(cloud='_test_cloud_', auth={'username': 'user'}) + cc = c.get_one_cloud(cloud='_test-cloud_', auth={'username': 'user'}) self.assertEqual('user', cc.auth['username']) self.assertEqual('token', cc.auth_type) self.assertEqual( @@ -66,7 +66,7 @@ class TestConfig(base.TestCase): self.assertIsInstance(c.cloud_config['cache'], dict) self.assertIn('max_age', c.cloud_config['cache']) self.assertIn('path', c.cloud_config['cache']) - cc = c.get_one_cloud('_test_cloud_') + cc = c.get_one_cloud('_test-cloud_') self._assert_cloud_details(cc) cc = c.get_one_cloud('_test_cloud_no_vendor') self._assert_cloud_details(cc) @@ -87,7 +87,7 @@ class TestConfig(base.TestCase): def test_get_one_cloud_auth_merge(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml]) - cc = c.get_one_cloud(cloud='_test_cloud_', auth={'username': 'user'}) + cc = c.get_one_cloud(cloud='_test-cloud_', auth={'username': 'user'}) self.assertEqual('user', cc.auth['username']) self.assertEqual('testpass', cc.auth['password']) @@ -106,7 +106,7 @@ class TestConfigArgparse(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud(cloud='_test_cloud_', argparse=self.options) + cc = c.get_one_cloud(cloud='_test-cloud_', argparse=self.options) self._assert_cloud_details(cc) self.assertEqual(cc.region_name, 'other-test-region') self.assertEqual(cc.snack_type, 'cookie') @@ -125,7 +125,7 @@ class TestConfigArgparse(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud(cloud='_test_cloud_', argparse=None) + cc = c.get_one_cloud(cloud='_test-cloud_', argparse=None) self._assert_cloud_details(cc) self.assertEqual(cc.region_name, 'test-region') self.assertIsNone(cc.snack_type) @@ -136,6 +136,6 @@ class TestConfigDefault(base.TestCase): def test_set_no_default(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud(cloud='_test_cloud_', argparse=None) + cc = c.get_one_cloud(cloud='_test-cloud_', argparse=None) self._assert_cloud_details(cc) self.assertEqual(cc.auth_type, 'password') diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index e60c73b31..365ad6780 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -59,7 +59,7 @@ class TestEnviron(base.TestCase): self.assertNotIn('auth_url', cc.config) self.assertIn('auth_url', cc.config['auth']) self.assertNotIn('auth_url', cc.config) - cc = c.get_one_cloud('_test_cloud_') + cc = c.get_one_cloud('_test-cloud_') self._assert_cloud_details(cc) cc = c.get_one_cloud('_test_cloud_no_vendor') self._assert_cloud_details(cc) @@ -72,7 +72,7 @@ class TestEnviron(base.TestCase): self.assertIsInstance(c.cloud_config['cache'], dict) self.assertIn('max_age', c.cloud_config['cache']) self.assertIn('path', c.cloud_config['cache']) - cc = c.get_one_cloud('_test_cloud_') + cc = c.get_one_cloud('_test-cloud_') self._assert_cloud_details(cc) cc = c.get_one_cloud('_test_cloud_no_vendor') self._assert_cloud_details(cc) From 6daed957c3647efb3d47d7e85065a33a74e39047 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Fri, 22 May 2015 11:25:37 -0700 Subject: [PATCH 081/365] Add flag to indicate handling of security groups Security groups can be handled by either nova, neutron, or just not be supported. This adds a flag for that. Values will be one of 'neutron' (the default), 'nova', or None. None indicates that security groups are not supported. Change-Id: I81a2e10e14e53acc9ffc328771d8ef721e2fd370 --- doc/source/vendor-support.rst | 6 ++++++ os_client_config/defaults.py | 1 + os_client_config/vendors.py | 1 + 3 files changed, 8 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index dea27d13d..02e093729 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -24,6 +24,7 @@ region-b.geo-1 US East * Image API Version is 1 * Images must be in `qcow2` format * Floating IPs are provided by Neutron +* Security groups are provided by Neutron Rackspace --------- @@ -46,6 +47,7 @@ HKG Hong Kong * Images must be in `vhd` format * Images must be uploaded using the Glance Task Interface * Floating IPs are not needed +* Security groups are not supported Dreamhost --------- @@ -61,6 +63,7 @@ RegionOne Region One * Image API Version is 2 * Images must be in `raw` format * Floating IPs are provided by Neutron +* Security groups are provided by Neutron Vexxhost -------- @@ -76,6 +79,7 @@ ca-ymq-1 Montreal * Image API Version is 2 * Images must be in `qcow2` format * Floating IPs are not needed +* Security groups are provided by Neutron RunAbove -------- @@ -92,6 +96,7 @@ BHS-1 Beauharnois, QC * Image API Version is 2 * Images must be in `qcow2` format * Floating IPs are not needed +* Security groups are provided by Neutron UnitedStack ----------- @@ -109,3 +114,4 @@ gd1 Guangdong * Image API Version is 2 * Images must be in `raw` format * Floating IPs are not needed +* Security groups are provided by Neutron diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index 14b820949..11a94c47a 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -23,6 +23,7 @@ _defaults = dict( image_api_version='1', network_api_version='2', object_api_version='1', + secgroup_source='neutron', volume_api_version='1', ) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index dec63dc2e..ed8d0471e 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -33,6 +33,7 @@ CLOUD_DEFAULTS = dict( image_api_use_tasks=True, image_format='vhd', floating_ip_source=None, + secgroup_source=None, ), dreamhost=dict( auth=dict( From b16d6d391779cc0a1bb156f0cf4468521f4e5d4e Mon Sep 17 00:00:00 2001 From: TerryHowe Date: Wed, 27 May 2015 14:34:27 -0600 Subject: [PATCH 082/365] Add tests for get_cloud_names Change-Id: I21b71657d83fb25628f230a48ddca197570a38a9 --- os_client_config/tests/test_config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 5a7fc1a94..dddaf8b0f 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -91,6 +91,18 @@ class TestConfig(base.TestCase): self.assertEqual('user', cc.auth['username']) self.assertEqual('testpass', cc.auth['password']) + def test_get_cloud_names(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml]) + self.assertEqual(['_test-cloud_', '_test_cloud_no_vendor'], + sorted(c.get_cloud_names())) + c = config.OpenStackConfig(config_files=[self.no_yaml], + vendor_files=[self.no_yaml]) + for k in os.environ.keys(): + if k.startswith('OS_'): + self.useFixture(fixtures.EnvironmentVariable(k)) + c.get_one_cloud(cloud='defaults') + self.assertEqual(['defaults'], sorted(c.get_cloud_names())) + class TestConfigArgparse(base.TestCase): From daab3615e2884100f88a797570c1326c8139983c Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Thu, 14 May 2015 20:01:38 +0000 Subject: [PATCH 083/365] Add set_one_cloud method It is useful for clients to be able to update configuration as well. By doing this in shade we can perform things like merging and do some testing rather than require clients to do it. Change-Id: Ia185847c29c10f2cc6838adf962defd80894d0db --- os_client_config/config.py | 31 ++++++++++++++++++++++ os_client_config/tests/test_config.py | 37 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 7b52912ed..b323e7028 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -424,6 +424,37 @@ class OpenStackConfig(object): name=cloud_name, region=config['region_name'], config=self._normalize_keys(config)) + @staticmethod + def set_one_cloud(config_file, cloud, set_config=None): + """Set a single cloud configuration. + + :param string config_file: + The path to the config file to edit. If this file does not exist + it will be created. + :param string cloud: + The name of the configuration to save to clouds.yaml + :param dict set_config: Configuration options to be set + """ + + set_config = set_config or {} + cur_config = {} + try: + with open(config_file) as fh: + cur_config = yaml.safe_load(fh) + except IOError as e: + # Not no such file + if e.errno != 2: + raise + pass + + clouds_config = cur_config.get('clouds', {}) + cloud_config = _auth_update(clouds_config.get(cloud, {}), set_config) + clouds_config[cloud] = cloud_config + cur_config['clouds'] = clouds_config + + with open(config_file, 'w') as fh: + yaml.safe_dump(cur_config, fh, default_flow_style=False) + if __name__ == '__main__': config = OpenStackConfig().get_all_clouds() for cloud in config: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index dddaf8b0f..25e1cc199 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -13,9 +13,11 @@ # under the License. import argparse +import copy import os import fixtures +import yaml from os_client_config import cloud_config from os_client_config import config @@ -103,6 +105,41 @@ class TestConfig(base.TestCase): c.get_one_cloud(cloud='defaults') self.assertEqual(['defaults'], sorted(c.get_cloud_names())) + def test_set_one_cloud_creates_file(self): + config_dir = fixtures.TempDir() + self.useFixture(config_dir) + config_path = os.path.join(config_dir.path, 'clouds.yaml') + config.OpenStackConfig.set_one_cloud(config_path, '_test_cloud_') + self.assertTrue(os.path.isfile(config_path)) + with open(config_path) as fh: + self.assertEqual({'clouds': {'_test_cloud_': {}}}, + yaml.safe_load(fh)) + + def test_set_one_cloud_updates_cloud(self): + new_config = { + 'cloud': 'new_cloud', + 'auth': { + 'password': 'newpass' + } + } + + resulting_cloud_config = { + 'auth': { + 'password': 'newpass', + 'username': 'testuser' + }, + 'cloud': 'new_cloud', + 'profile': '_test_cloud_in_our_cloud', + 'region_name': 'test-region' + } + resulting_config = copy.deepcopy(base.USER_CONF) + resulting_config['clouds']['_test-cloud_'] = resulting_cloud_config + config.OpenStackConfig.set_one_cloud(self.cloud_yaml, '_test-cloud_', + new_config) + with open(self.cloud_yaml) as fh: + written_config = yaml.safe_load(fh) + self.assertEqual(written_config, resulting_config) + class TestConfigArgparse(base.TestCase): From d503f0594fbdc377c07e4d79ce2cb08f6226aa7d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 29 May 2015 07:39:20 -0400 Subject: [PATCH 084/365] Add list of image params needed to disable agents Some clouds have an in-instance agent to handle things like online password resets and other such activities. When building and uploading images, it's often advantageous to not install such an agent and instead handle such things via config management... but doing so requires data to be set on the image itself. Change-Id: I5b7c9d72fd2d49890bc466d7dd22a3cb9595f670 --- doc/source/vendor-support.rst | 3 +++ os_client_config/defaults.py | 1 + os_client_config/vendors.py | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 02e093729..602dd2bc0 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -48,6 +48,9 @@ HKG Hong Kong * Images must be uploaded using the Glance Task Interface * Floating IPs are not needed * Security groups are not supported +* Uploaded Images need properties to not use vendor agent +:vm_mode: hvm +:xenapi_use_agent: False Dreamhost --------- diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index 11a94c47a..e92149223 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -25,6 +25,7 @@ _defaults = dict( object_api_version='1', secgroup_source='neutron', volume_api_version='1', + disable_vendor_agent={}, ) diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index ed8d0471e..68a70c544 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -34,6 +34,10 @@ CLOUD_DEFAULTS = dict( image_format='vhd', floating_ip_source=None, secgroup_source=None, + disable_vendor_agent=dict( + vm_mode='hvm', + xenapi_use_agent=False, + ) ), dreamhost=dict( auth=dict( From f66aad5ad9b3ac3476d591c71546b468ab1407d3 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 30 May 2015 13:01:41 -0400 Subject: [PATCH 085/365] Add auro to list of known vendors Change-Id: I48d6cffdcc16bdbf4912da775fb29876007fe8db --- doc/source/vendor-support.rst | 17 +++++++++++++++++ os_client_config/vendors.py | 11 +++++++++++ 2 files changed, 28 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 602dd2bc0..1818e6b5f 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -118,3 +118,20 @@ gd1 Guangdong * Images must be in `raw` format * Floating IPs are not needed * Security groups are provided by Neutron + +Auro +---- + +https://api.auro.io:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +RegionOne RegionOne +============== ================ + +* Identity API Version is 2 +* Image API Version is 1 +* Images must be in `qcow2` format +* Floating IPs are provided by Nova +* Security groups are provided by Nova diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index 68a70c544..ad04c4dc1 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -73,4 +73,15 @@ CLOUD_DEFAULTS = dict( image_format='raw', floating_ip_source=None, ), + auro=dict( + auth=dict( + auth_url='https://api.auro.io:5000/v2.0', + ), + region_name='RegionOne', + identity_api_version='2', + image_api_version='1', + image_format='qcow2', + secgroup_source='nova', + floating_ip_source='nova', + ), ) From b431669a6b8210668fa37751bd208b55720ba314 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 30 May 2015 13:02:47 -0400 Subject: [PATCH 086/365] Change naming in vendor doc to match vendors.py Change-Id: I90da039d8b0551b80ec34975480e650885f59d3a --- doc/source/vendor-support.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 1818e6b5f..351d66665 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -8,8 +8,8 @@ information about various things a user would need to know. The following is a text representation of the vendor related defaults `os-client-config` knows about. -HP Cloud --------- +hp +-- https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 @@ -26,7 +26,7 @@ region-b.geo-1 US East * Floating IPs are provided by Neutron * Security groups are provided by Neutron -Rackspace +rackspace --------- https://identity.api.rackspacecloud.com/v2.0/ @@ -52,7 +52,7 @@ HKG Hong Kong :vm_mode: hvm :xenapi_use_agent: False -Dreamhost +dreamhost --------- https://keystone.dream.io/v2.0 @@ -68,7 +68,7 @@ RegionOne Region One * Floating IPs are provided by Neutron * Security groups are provided by Neutron -Vexxhost +vexxhost -------- http://auth.api.thenebulacloud.com:5000/v2.0/ @@ -84,7 +84,7 @@ ca-ymq-1 Montreal * Floating IPs are not needed * Security groups are provided by Neutron -RunAbove +runabove -------- https://auth.runabove.io/v2.0 @@ -101,7 +101,7 @@ BHS-1 Beauharnois, QC * Floating IPs are not needed * Security groups are provided by Neutron -UnitedStack +unitedstack ----------- https://identity.api.ustack.com/v3 @@ -119,7 +119,7 @@ gd1 Guangdong * Floating IPs are not needed * Security groups are provided by Neutron -Auro +auro ---- https://api.auro.io:5000/v2.0 From ba0986f58ce7468bdc91380ecd1b07086fa37964 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 2 Jun 2015 15:55:59 -0500 Subject: [PATCH 087/365] Add more defaults to our defaults file It's easier logic when we have default values for things. Change-Id: I2d66dcee68ea95371609640677fd41cca4b0a7cf --- os_client_config/defaults.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index e92149223..8bf693d8d 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -13,10 +13,12 @@ # under the License. _defaults = dict( + api_timeout=None, auth_type='password', baremetal_api_version='1', compute_api_version='2', database_api_version='1.0', + endpoint_type='public', floating_ip_source='neutron', identity_api_version='2', image_api_use_tasks=False, @@ -26,6 +28,11 @@ _defaults = dict( secgroup_source='neutron', volume_api_version='1', disable_vendor_agent={}, + # SSL Related args + verify=True, + cacert=None, + cert=None, + key=None, ) From a0df67704ace186b18fd1ecdc220a7e56409bc6f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 2 Jun 2015 15:56:17 -0500 Subject: [PATCH 088/365] Provide a helper method to get requests ssl values There is a weird logic around the interaction between how the openstack things all accept cacert and verify/insecure parameters and how requests wants them. Rather than spreading the parameter combining logic across the universe, put it here. Note that this inverts the usual requests logic in that !verify will override the presence of a cacert value and cause verification to NOT occur. This is intended to become the normal mode of operation for OpenStack clients. Change-Id: I3c76d9a10e6e5d60a593ceefc87dafdc6857d9c6 --- os_client_config/cloud_config.py | 13 +++++++++ os_client_config/tests/test_cloud_config.py | 29 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 347eb7c48..d8ac85d55 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -39,3 +39,16 @@ class CloudConfig(object): def __ne__(self, other): return not self == other + + def get_requests_verify_args(self): + """Return the verify and cert values for the requests library.""" + if self.config['verify'] and self.config['cacert']: + verify = self.config['cacert'] + else: + verify = self.config['verify'] + + cert = self.config.get('cert', None) + if cert: + if self.config['key']: + cert = (cert, self.config['key']) + return (verify, cert) diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 1f79b3e71..729bc9974 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import copy from os_client_config import cloud_config from os_client_config.tests import base @@ -59,3 +60,31 @@ class TestCloudConfig(base.TestCase): cc2 = cloud_config.CloudConfig("test1", "region-al", {}) self.assertNotEqual(cc1, cc2) + + def test_verify(self): + config_dict = copy.deepcopy(fake_config_dict) + config_dict['cacert'] = None + + config_dict['verify'] = False + cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) + (verify, cacert) = cc.get_requests_verify_args() + self.assertFalse(verify) + + config_dict['verify'] = True + cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) + (verify, cacert) = cc.get_requests_verify_args() + self.assertTrue(verify) + + def test_verify_cacert(self): + config_dict = copy.deepcopy(fake_config_dict) + config_dict['cacert'] = "certfile" + + config_dict['verify'] = False + cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) + (verify, cacert) = cc.get_requests_verify_args() + self.assertFalse(verify) + + config_dict['verify'] = True + cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) + (verify, cacert) = cc.get_requests_verify_args() + self.assertEqual("certfile", verify) From 038ddd38185ace24e96017b3d638952a397e61c6 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Thu, 4 Jun 2015 09:56:33 +0200 Subject: [PATCH 089/365] Add cloud vendor files config in doc Although it is possible to include specific config files for unknown vendors, this wasn't specified in the documentation. Change-Id: Ib27277d480e373a8a083e820161e0bdb985de284 --- README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 193f0a112..ed232c49e 100644 --- a/README.rst +++ b/README.rst @@ -108,10 +108,12 @@ An example config file is probably helpful: You may note a few things. First, since auth_url settings are silly and embarrasingly ugly, known cloud vendor profile information is included and -may be referrenced by name. One of the benefits of that is that auth_url +may be referenced by name. One of the benefits of that is that auth_url isn't the only thing the vendor defaults contain. For instance, since Rackspace lists `rax:database` as the service type for trove, os-client-config -knows that so that you don't have to. +knows that so that you don't have to. In case the cloud vendor profile is not +available, you can provide one called clouds-public.yaml, following the same +location rules previously mentioned for the config files. Also, region_name can be a list of regions. When you call get_all_clouds, you'll get a cloud config object for each cloud/region combo. From acc2cbdc98a80c71be4e1f55580914b9fdf8d4ab Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Thu, 4 Jun 2015 10:24:58 +0200 Subject: [PATCH 090/365] Raise a warning when using 'cloud' in config The former use of 'cloud' in the config file, was changed in favor of 'profile' to avoid confusions. Change-Id: Iba08746a06ebb397ee1d0f59d5cda41db2b86053 --- os_client_config/config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index b323e7028..fa1843382 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -14,6 +14,7 @@ import os +import warnings import appdirs import yaml @@ -217,6 +218,11 @@ class OpenStackConfig(object): # for this. profile_name = our_cloud.get('profile', our_cloud.get('cloud', None)) if profile_name: + if 'cloud' in our_cloud: + warnings.warn( + "clouds.yaml use the keyword 'cloud' to reference a known " + "vendor profile. This has been deprecated in favor of the " + "'profile' keyword.") vendor_filename, vendor_file = self._load_vendor_file() if vendor_file and profile_name in vendor_file['public-clouds']: _auth_update(cloud, vendor_file['public-clouds'][profile_name]) From b1e562c3e59f64118d184e8655c7bfe343a09cdc Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Thu, 4 Jun 2015 11:21:38 +0200 Subject: [PATCH 091/365] Change references of "clouds.yaml" for real file The config file clouds.yaml can be located in several places, and even the file extension can be .yml. Replace all user visible messages making reference for that file to show the full path of the config file used. Change-Id: I489d87368b72dfe69b7d4e3c07ba5d5249c45667 --- os_client_config/config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index fa1843382..dd80e2a72 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -120,10 +120,11 @@ class OpenStackConfig(object): self.envvar_key = os.environ.pop('OS_CLOUD_NAME', 'envvars') if self.envvar_key in self.cloud_config['clouds']: raise exceptions.OpenStackConfigException( - 'clouds.yaml defines a cloud named "{0}", but' - ' OS_CLOUD_NAME is also set to "{0}". Please rename' + '"{0}" defines a cloud named "{1}", but' + ' OS_CLOUD_NAME is also set to "{1}". Please rename' ' either your environment based cloud, or one of your' - ' file-based clouds.'.format(self.envvar_key)) + ' file-based clouds.'.format(self.config_filename, + self.envvar_key)) envvars = _get_os_environ() if envvars: @@ -220,9 +221,9 @@ class OpenStackConfig(object): if profile_name: if 'cloud' in our_cloud: warnings.warn( - "clouds.yaml use the keyword 'cloud' to reference a known " + "{0} use the keyword 'cloud' to reference a known " "vendor profile. This has been deprecated in favor of the " - "'profile' keyword.") + "'profile' keyword.".format(self.config_filename)) vendor_filename, vendor_file = self._load_vendor_file() if vendor_file and profile_name in vendor_file['public-clouds']: _auth_update(cloud, vendor_file['public-clouds'][profile_name]) From bc253d62b9a61575c3ba2f443b2429b5df904905 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Thu, 4 Jun 2015 13:06:43 +0200 Subject: [PATCH 092/365] Raise a warning with conflicting SSL params Setting a cacert to check the cloud cert is useless when changing the default verify flag to False since this will have precedence. Raise a warning to alert the user about this behavior. Change-Id: I099d03fef5e8da0d6eed572613f4693604173ecd --- os_client_config/cloud_config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index d8ac85d55..018908e92 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import warnings + class CloudConfig(object): def __init__(self, name, region, config): @@ -46,6 +48,11 @@ class CloudConfig(object): verify = self.config['cacert'] else: verify = self.config['verify'] + if self.config['cacert']: + warnings.warn( + "You are specifying a cacert for the cloud {0} but " + "also to ignore the host verification. The host SSL cert " + "will not be verified.".format(self.name)) cert = self.config.get('cert', None) if cert: From 95beafeadd99383fc99dd8aff5bfc9ce8ece7030 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 5 Jun 2015 10:40:01 -0400 Subject: [PATCH 093/365] Stringify project details There are some clouds that have things like integer project names. That's no fun when they are interperted at int, so stringify them. Stringifying integer project ids is apparently less important, but does not hurt. Change-Id: Ife9ecaa28c552d589dbea9a065da0dfa483592eb --- os_client_config/config.py | 4 ++-- os_client_config/tests/base.py | 8 ++++++++ os_client_config/tests/test_config.py | 14 ++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index b323e7028..432d1b470 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -251,10 +251,10 @@ class OpenStackConfig(object): target = None for key in possible_values: if key in cloud: - target = cloud[key] + target = str(cloud[key]) del cloud[key] if key in cloud['auth']: - target = cloud['auth'][key] + target = str(cloud['auth'][key]) del cloud['auth'][key] if target: cloud['auth'][target_key] = target diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 6a412bfd6..967f119dd 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -56,6 +56,14 @@ USER_CONF = { }, 'region-name': 'test-region', }, + '_test-cloud-int-project_': { + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project_id': 12345, + }, + 'region_name': 'test-region', + }, }, 'cache': {'max_age': 1}, } diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 25e1cc199..562e40815 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -73,6 +73,12 @@ class TestConfig(base.TestCase): cc = c.get_one_cloud('_test_cloud_no_vendor') self._assert_cloud_details(cc) + def test_get_one_cloud_with_int_project_id(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test-cloud-int-project_') + self.assertEqual('12345', cc.auth['project_name']) + def test_no_environ(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) @@ -95,8 +101,12 @@ class TestConfig(base.TestCase): def test_get_cloud_names(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml]) - self.assertEqual(['_test-cloud_', '_test_cloud_no_vendor'], - sorted(c.get_cloud_names())) + self.assertEqual( + ['_test-cloud-int-project_', + '_test-cloud_', + '_test_cloud_no_vendor', + ], + sorted(c.get_cloud_names())) c = config.OpenStackConfig(config_files=[self.no_yaml], vendor_files=[self.no_yaml]) for k in os.environ.keys(): From 7e605f963fe88eded0017a4fdf85ebf13b4c52c1 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Thu, 4 Jun 2015 13:39:12 +0200 Subject: [PATCH 094/365] Add SSL documentation to README.rst Explain usage and warn avoid behavior with conflicting cacert and verify options. Change-Id: I25b43ba47bd0feb941b649265c6e67723a93e277 --- README.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.rst b/README.rst index ed232c49e..4d3f50cb7 100644 --- a/README.rst +++ b/README.rst @@ -135,6 +135,20 @@ as a result of a chosen plugin need to go into the auth dict. For password auth, this includes `auth_url`, `username` and `password` as well as anything related to domains, projects and trusts. +SSL Settings +------------ + +When the access to a cloud is done via a secure connection, `os-client-config` +will always verify the SSL cert by default. This can be disabled by setting +`verify` to `False`. In case the cert is signed by an unknown CA, a specific +cacert can be provided via `cacert`. **WARNING:** `verify` will always have +precedence over `cacert`, so when setting a CA cert but disabling `verify`, the +cloud cert will never be validated. + +Client certs are also configurable. `cert` will be the client cert file +location. In case the cert key is not included within the client cert file, +its file location needs to be set via `key`. + Cache Settings -------------- From a2f25244d352f3f537f602f6a4b8f0ca785c05b6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 5 Jun 2015 15:48:17 -0400 Subject: [PATCH 095/365] Add support for OVH Public Cloud Change-Id: If2b4bc34a159d1ef4180fbd5de9bbedfaa5b3e82 --- doc/source/vendor-support.rst | 17 +++++++++++++++++ os_client_config/vendors.py | 11 +++++++++++ 2 files changed, 28 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 351d66665..e7d51c01a 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -135,3 +135,20 @@ RegionOne RegionOne * Images must be in `qcow2` format * Floating IPs are provided by Nova * Security groups are provided by Nova + +ovh +--- + +https://auth.cloud.ovh.net/v2.0 + +============== ================ +Region Name Human Name +============== ================ +SBG-1 Strassbourg, FR +============== ================ + +* Identity API Version is 2 +* Image API Version is 1 +* Images must be in `raw` format +* Floating IPs are provided by Neutron +* Security groups are provided by Neutron diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py index ad04c4dc1..e02b32517 100644 --- a/os_client_config/vendors.py +++ b/os_client_config/vendors.py @@ -84,4 +84,15 @@ CLOUD_DEFAULTS = dict( secgroup_source='nova', floating_ip_source='nova', ), + ovh=dict( + auth=dict( + auth_url='https://auth.cloud.ovh.net/v2.0', + ), + region_name='SBG1', + identity_api_version='2', + image_api_version='1', + image_format='raw', + secgroup_source='neutron', + floating_ip_source='neutron', + ), ) From d710accb3fd95eb7e5bc108a484e3ecc78498af7 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Sat, 6 Jun 2015 13:32:20 +0200 Subject: [PATCH 096/365] Some cleanup in the README.rst Change-Id: I9f7c6c727708a9095566bce8d4ff03837be95d07 --- README.rst | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index 4d3f50cb7..9cdda45dc 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ os-client-config =============================== -os-client-config is a library for collecting client configuration for +`os-client-config` is a library for collecting client configuration for using an OpenStack cloud in a consistent and comprehensive manner. It will find cloud config for as few as 1 cloud and as many as you want to put in a config file. It will read environment variables and config files, @@ -10,19 +10,19 @@ and it also contains some vendor specific default values so that you don't have to know extra info to use OpenStack * If you have a config file, you will get the clouds listed in it -* If you have environment variables, you will get a cloud named 'envvars' -* If you have neither, you will get a cloud named 'defaults' with base defaults +* If you have environment variables, you will get a cloud named `envvars` +* If you have neither, you will get a cloud named `defaults` with base defaults Environment Variables --------------------- -os-client-config honors all of the normal `OS_*` variables. It does not +`os-client-config` honors all of the normal `OS_*` variables. It does not provide backwards compatibility to service-specific variables such as `NOVA_USERNAME`. -If you have OpenStack environment variables set, os-client-config will produce -a cloud config object named "envvars" containing your values from the -environment. If you don't like the name "envvars", that's ok, you can override +If you have OpenStack environment variables set, `os-client-config` will produce +a cloud config object named `envvars` containing your values from the +environment. If you don't like the name `envvars`, that's ok, you can override it by setting `OS_CLOUD_NAME`. Service specific settings, like the nova service type, are set with the @@ -34,7 +34,7 @@ for trove set:: Config Files ------------ -os-client-config will look for a file called clouds.yaml in the following +`os-client-config` will look for a file called `clouds.yaml` in the following locations: * Current Directory @@ -50,6 +50,11 @@ Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type for trove (because you're using Rackspace) set: +:: + + database_service_type: 'rax:database' + + Site Specific File Locations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -72,10 +77,6 @@ look in an OS specific config dir * OSX: `/Library/Application Support/openstack` * Windows: `C:\\ProgramData\\OpenStack\\openstack` -:: - - database_service_type: 'rax:database' - An example config file is probably helpful: :: @@ -106,16 +107,16 @@ An example config file is probably helpful: project_id: 610275 region_name: DFW,ORD,IAD -You may note a few things. First, since auth_url settings are silly +You may note a few things. First, since `auth_url` settings are silly and embarrasingly ugly, known cloud vendor profile information is included and -may be referenced by name. One of the benefits of that is that auth_url +may be referenced by name. One of the benefits of that is that `auth_url` isn't the only thing the vendor defaults contain. For instance, since -Rackspace lists `rax:database` as the service type for trove, os-client-config +Rackspace lists `rax:database` as the service type for trove, `os-client-config` knows that so that you don't have to. In case the cloud vendor profile is not -available, you can provide one called clouds-public.yaml, following the same +available, you can provide one called `clouds-public.yaml`, following the same location rules previously mentioned for the config files. -Also, region_name can be a list of regions. When you call get_all_clouds, +Also, `region_name` can be a list of regions. When you call `get_all_clouds`, you'll get a cloud config object for each cloud/region combo. As seen with `dns_service_type`, any setting that makes sense to be per-service, @@ -153,7 +154,7 @@ Cache Settings -------------- Accessing a cloud is often expensive, so it's quite common to want to do some -client-side caching of those operations. To facilitate that, os-client-config +client-side caching of those operations. To facilitate that, `os-client-config` understands passing through cache settings to dogpile.cache, with the following behaviors: From 9f7a50619a170ec9c9d702862ea617cc1b17e765 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Sat, 6 Jun 2015 12:50:42 +0200 Subject: [PATCH 097/365] Raise warning when a vendor profile is missing Change-Id: I12f5c9d824abee4af42403a86db8bf4a8bbcac16 --- os_client_config/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index c08fbeea9..bc8ab1e33 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -232,7 +232,9 @@ class OpenStackConfig(object): _auth_update(cloud, vendors.CLOUD_DEFAULTS[profile_name]) except KeyError: # Can't find the requested vendor config, go about business - pass + warnings.warn("Couldn't find the vendor profile '{0}', for" + " the cloud '{1}'".format(profile_name, + name)) if 'auth' not in cloud: cloud['auth'] = dict() From 843402653d8c05698db481c4146a67846219ae60 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Sat, 6 Jun 2015 13:06:39 +0200 Subject: [PATCH 098/365] Use one yaml file per vendor With an increasing numbers of vendors, having all profiles in a single file, and with dicts inside dicts inside dicts... is getting more complicated to have a general view of what is available. Change-Id: I6f386829774365125d585a6ff1b6e22f4c98df2a --- os_client_config/vendors.py | 98 ----------------------- os_client_config/vendors/__init__.py | 25 ++++++ os_client_config/vendors/auro.yaml | 10 +++ os_client_config/vendors/dreamhost.yaml | 7 ++ os_client_config/vendors/hp.yaml | 8 ++ os_client_config/vendors/ovh.yaml | 10 +++ os_client_config/vendors/rackspace.yaml | 14 ++++ os_client_config/vendors/runabove.yaml | 7 ++ os_client_config/vendors/unitedstack.yaml | 8 ++ os_client_config/vendors/vexxhost.yaml | 8 ++ 10 files changed, 97 insertions(+), 98 deletions(-) delete mode 100644 os_client_config/vendors.py create mode 100644 os_client_config/vendors/__init__.py create mode 100644 os_client_config/vendors/auro.yaml create mode 100644 os_client_config/vendors/dreamhost.yaml create mode 100644 os_client_config/vendors/hp.yaml create mode 100644 os_client_config/vendors/ovh.yaml create mode 100644 os_client_config/vendors/rackspace.yaml create mode 100644 os_client_config/vendors/runabove.yaml create mode 100644 os_client_config/vendors/unitedstack.yaml create mode 100644 os_client_config/vendors/vexxhost.yaml diff --git a/os_client_config/vendors.py b/os_client_config/vendors.py deleted file mode 100644 index e02b32517..000000000 --- a/os_client_config/vendors.py +++ /dev/null @@ -1,98 +0,0 @@ -# flake8: noqa -# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -CLOUD_DEFAULTS = dict( - hp=dict( - auth=dict( - auth_url='https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0', - ), - region_name='region-b.geo-1', - dns_service_type='hpext:dns', - image_api_version='1', - image_format='qcow2', - ), - rackspace=dict( - auth=dict( - auth_url='https://identity.api.rackspacecloud.com/v2.0/', - ), - database_service_type='rax:database', - compute_service_name='cloudServersOpenStack', - image_api_version='2', - image_api_use_tasks=True, - image_format='vhd', - floating_ip_source=None, - secgroup_source=None, - disable_vendor_agent=dict( - vm_mode='hvm', - xenapi_use_agent=False, - ) - ), - dreamhost=dict( - auth=dict( - auth_url='https://keystone.dream.io/v2.0', - ), - region_name='RegionOne', - image_api_version='2', - image_format='raw', - ), - vexxhost=dict( - auth=dict( - auth_url='http://auth.api.thenebulacloud.com:5000/v2.0/', - ), - region_name='ca-ymq-1', - image_api_version='2', - image_format='qcow2', - floating_ip_source=None, - ), - runabove=dict( - auth=dict( - auth_url='https://auth.runabove.io/v2.0', - ), - image_api_version='2', - image_format='qcow2', - floating_ip_source=None, - ), - unitedstack=dict( - auth=dict( - auth_url='https://identity.api.ustack.com/v3', - ), - identity_api_version='3', - image_api_version='2', - image_format='raw', - floating_ip_source=None, - ), - auro=dict( - auth=dict( - auth_url='https://api.auro.io:5000/v2.0', - ), - region_name='RegionOne', - identity_api_version='2', - image_api_version='1', - image_format='qcow2', - secgroup_source='nova', - floating_ip_source='nova', - ), - ovh=dict( - auth=dict( - auth_url='https://auth.cloud.ovh.net/v2.0', - ), - region_name='SBG1', - identity_api_version='2', - image_api_version='1', - image_format='raw', - secgroup_source='neutron', - floating_ip_source='neutron', - ), -) diff --git a/os_client_config/vendors/__init__.py b/os_client_config/vendors/__init__.py new file mode 100644 index 000000000..0ccb04c5d --- /dev/null +++ b/os_client_config/vendors/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import glob +import os + +import yaml + +vendors_path = os.path.dirname(os.path.realpath(__file__)) + +CLOUD_DEFAULTS = {} +for vendor in glob.glob(os.path.join(vendors_path, '*.yaml')): + with open(vendor, 'r') as f: + CLOUD_DEFAULTS.update(yaml.safe_load(f)) diff --git a/os_client_config/vendors/auro.yaml b/os_client_config/vendors/auro.yaml new file mode 100644 index 000000000..da1491dec --- /dev/null +++ b/os_client_config/vendors/auro.yaml @@ -0,0 +1,10 @@ +--- +auro: + auth: + auth_url: https://api.auro.io:5000/v2.0 + region_name: RegionOne + identity_api_version: 2 + image_api_version: 1 + image_format: qcow2 + secgroup_source: nova + floating_ip_source: nova diff --git a/os_client_config/vendors/dreamhost.yaml b/os_client_config/vendors/dreamhost.yaml new file mode 100644 index 000000000..63f15c31d --- /dev/null +++ b/os_client_config/vendors/dreamhost.yaml @@ -0,0 +1,7 @@ +--- +dreamhost: + auth: + auth_url: https://keystone.dream.io/v2.0 + region_name: RegionOne + image_api_version: 2 + image_format: raw diff --git a/os_client_config/vendors/hp.yaml b/os_client_config/vendors/hp.yaml new file mode 100644 index 000000000..2ac551781 --- /dev/null +++ b/os_client_config/vendors/hp.yaml @@ -0,0 +1,8 @@ +--- +hp: + auth: + auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 + region_name: region-b.geo-1 + dns_service_type: hpext:dns + image_api_version: 1 + image_format: qcow2 diff --git a/os_client_config/vendors/ovh.yaml b/os_client_config/vendors/ovh.yaml new file mode 100644 index 000000000..5b4426f18 --- /dev/null +++ b/os_client_config/vendors/ovh.yaml @@ -0,0 +1,10 @@ +--- +ovh: + auth: + auth_url: https://auth.cloud.ovh.net/v2.0 + region_name: SBG1 + identity_api_version: 2 + image_api_version: 1 + image_format: raw + secgroup_source: neutron + floating_ip_source: neutron diff --git a/os_client_config/vendors/rackspace.yaml b/os_client_config/vendors/rackspace.yaml new file mode 100644 index 000000000..7af3cab96 --- /dev/null +++ b/os_client_config/vendors/rackspace.yaml @@ -0,0 +1,14 @@ +--- +rackspace: + auth: + auth_url: https://identity.api.rackspacecloud.com/v2.0 + database_service_type: 'rax:database' + compute_service_name: cloudServersOpenStack + image_api_version: 2 + image_api_use_tasks: True + image_format: vhd + floating_ip_source: None + secgroup_source: None + disable_vendor_agent: + vm_mode: hvm + xenapi_use_agent: False diff --git a/os_client_config/vendors/runabove.yaml b/os_client_config/vendors/runabove.yaml new file mode 100644 index 000000000..c9641dd41 --- /dev/null +++ b/os_client_config/vendors/runabove.yaml @@ -0,0 +1,7 @@ +--- +runabove: + auth: + auth_url: https://auth.runabove.io/v2.0 + image_api_version: 2 + image_format: qcow2 + floating_ip_sourc: None diff --git a/os_client_config/vendors/unitedstack.yaml b/os_client_config/vendors/unitedstack.yaml new file mode 100644 index 000000000..43c600a6f --- /dev/null +++ b/os_client_config/vendors/unitedstack.yaml @@ -0,0 +1,8 @@ +--- +unitedstack: + auth: + auth_url: https://identity.api.ustack.com/v3 + identity_api_version: 3 + image_api_version: 2 + image_format: raw + floating_ip_source: None diff --git a/os_client_config/vendors/vexxhost.yaml b/os_client_config/vendors/vexxhost.yaml new file mode 100644 index 000000000..5247ef436 --- /dev/null +++ b/os_client_config/vendors/vexxhost.yaml @@ -0,0 +1,8 @@ +--- +vexxhost: + auth: + auth_url: http://auth.api.thenebulacloud.com:5000/v2.0/ + region_name: ca-ymq-1 + image_api_version: 2 + image_format: qcow2 + floating_ip_source: None From de84b798f9eaac831621a63198ce2366effaeabd Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Tue, 9 Jun 2015 13:00:16 +0200 Subject: [PATCH 099/365] Add test to check cert and key as a tuple When the cert file doesn't include a key within it, a tuple with the cert and key needs to be passed to the requests library. Change-Id: I17534b8e7d07b3ad102cc6a6a839541c83281b8e --- os_client_config/tests/test_cloud_config.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 729bc9974..5f964dbeb 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -67,12 +67,12 @@ class TestCloudConfig(base.TestCase): config_dict['verify'] = False cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) - (verify, cacert) = cc.get_requests_verify_args() + (verify, cert) = cc.get_requests_verify_args() self.assertFalse(verify) config_dict['verify'] = True cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) - (verify, cacert) = cc.get_requests_verify_args() + (verify, cert) = cc.get_requests_verify_args() self.assertTrue(verify) def test_verify_cacert(self): @@ -81,10 +81,22 @@ class TestCloudConfig(base.TestCase): config_dict['verify'] = False cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) - (verify, cacert) = cc.get_requests_verify_args() + (verify, cert) = cc.get_requests_verify_args() self.assertFalse(verify) config_dict['verify'] = True cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) - (verify, cacert) = cc.get_requests_verify_args() + (verify, cert) = cc.get_requests_verify_args() self.assertEqual("certfile", verify) + + def test_cert_with_key(self): + config_dict = copy.deepcopy(fake_config_dict) + config_dict['cacert'] = None + config_dict['verify'] = False + + config_dict['cert'] = 'cert' + config_dict['key'] = 'key' + + cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) + (verify, cert) = cc.get_requests_verify_args() + self.assertEqual(("cert", "key"), cert) From eded4b31970d0762119548ef76e4f60dec2376d1 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Tue, 9 Jun 2015 17:49:30 +0200 Subject: [PATCH 100/365] Add missing tests - Test get_all_clouds returns a full list of available clouds - Test env. variables are stripped of initial 'os_' and '-' replaced with '_' Change-Id: If277aade17776d57236cc0e48a46fbb04158e7ed --- os_client_config/tests/test_config.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 562e40815..a6ba2ee83 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -28,6 +28,14 @@ from os_client_config.tests import base class TestConfig(base.TestCase): + def test_get_all_clouds(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + clouds = c.get_all_clouds() + user_clouds = [cloud for cloud in base.USER_CONF['clouds'].keys()] + configured_clouds = [cloud.name for cloud in clouds] + self.assertItemsEqual(user_clouds, configured_clouds) + def test_get_one_cloud(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) @@ -189,6 +197,15 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(cc.region_name, 'test-region') self.assertIsNone(cc.snack_type) + def test_fix_env_args(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + env_args = {'os-compute-api-version': 1} + fixed_args = c._fix_args(env_args) + + self.assertDictEqual({'compute_api_version': 1}, fixed_args) + class TestConfigDefault(base.TestCase): From 1550cfce0a11eb92f2411be9c94136e01c0a185b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 14 Jun 2015 04:18:32 +0300 Subject: [PATCH 101/365] Add some accessor methods to CloudConfig There are some questions that people might want to ask without necessarily digging in to the underlying dict (this came up when noodling on openstacksdk factory functions. Change-Id: I3d9554a5e64797794de646d4d0d61936b857f2b4 --- doc/source/vendor-support.rst | 42 +++++++++++++++++--------------- os_client_config/cloud_config.py | 37 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index e7d51c01a..314e36229 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -8,6 +8,22 @@ information about various things a user would need to know. The following is a text representation of the vendor related defaults `os-client-config` knows about. +Default Values +-------------- + +These are the default behaviors unless a cloud is configured differently. + +* Identity uses `password` authentication +* Identity API Version is 2 +* Image API Version is 1 +* Images must be in `qcow2` format +* Images are uploaded using PUT interface +* Public IPv4 is directly routable via DHCP from Neutron +* IPv6 is not provided +* Floating IPs are provided by Neutron +* Security groups are provided by Neutron +* Vendor specific agents are not used + hp -- @@ -21,10 +37,6 @@ region-b.geo-1 US East ============== ================ * DNS Service Type is `hpext:dns` -* Image API Version is 1 -* Images must be in `qcow2` format -* Floating IPs are provided by Neutron -* Security groups are provided by Neutron rackspace --------- @@ -47,6 +59,8 @@ HKG Hong Kong * Images must be in `vhd` format * Images must be uploaded using the Glance Task Interface * Floating IPs are not needed +* Public IPv4 is directly routable via static config by Nova +* IPv6 is provided to every server * Security groups are not supported * Uploaded Images need properties to not use vendor agent :vm_mode: hvm @@ -65,8 +79,8 @@ RegionOne Region One * Image API Version is 2 * Images must be in `raw` format -* Floating IPs are provided by Neutron -* Security groups are provided by Neutron +* Public IPv4 is provided via Floating IP from Neutron +* IPv6 is provided to every server vexxhost -------- @@ -80,9 +94,6 @@ ca-ymq-1 Montreal ============== ================ * Image API Version is 2 -* Images must be in `qcow2` format -* Floating IPs are not needed -* Security groups are provided by Neutron runabove -------- @@ -98,8 +109,7 @@ BHS-1 Beauharnois, QC * Image API Version is 2 * Images must be in `qcow2` format -* Floating IPs are not needed -* Security groups are provided by Neutron +* Floating IPs are not supported unitedstack ----------- @@ -116,8 +126,6 @@ gd1 Guangdong * Identity API Version is 3 * Image API Version is 2 * Images must be in `raw` format -* Floating IPs are not needed -* Security groups are provided by Neutron auro ---- @@ -131,8 +139,7 @@ RegionOne RegionOne ============== ================ * Identity API Version is 2 -* Image API Version is 1 -* Images must be in `qcow2` format +* Public IPv4 is provided via Floating IP from Nova * Floating IPs are provided by Nova * Security groups are provided by Nova @@ -147,8 +154,5 @@ Region Name Human Name SBG-1 Strassbourg, FR ============== ================ -* Identity API Version is 2 -* Image API Version is 1 * Images must be in `raw` format -* Floating IPs are provided by Neutron -* Security groups are provided by Neutron +* Floating IPs are not supported diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 018908e92..19f1bb4ee 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -59,3 +59,40 @@ class CloudConfig(object): if self.config['key']: cert = (cert, self.config['key']) return (verify, cert) + + def get_services(self): + """Return a list of service types we know something about.""" + services = [] + for key, val in self.config.items(): + if (key.endswith('api_version') + or key.endswith('service_type') + or key.endswith('service_name')): + services.append("_".join(key.split('_')[:-2])) + return list(set(services)) + + def get_auth_args(self): + return self.config['auth'] + + def get_endpoint_type(self, service_type=None): + if not service_type: + return self.config['endpoint_type'] + key = '{service_type}_endpoint_type'.format(service_type=service_type) + return self.config.get(key, self.config['endpoint_type']) + + def get_region_name(self, service_type=None): + if not service_type: + return self.region + key = '{service_type}_region_name'.format(service_type=service_type) + return self.config.get(key, self.region) + + def get_api_version(self, service_type): + key = '{service_type}_api_version'.format(service_type=service_type) + return self.config.get(key, None) + + def get_service_type(self, service_type): + key = '{service_type}_service_type'.format(service_type=service_type) + return self.config.get(key, service_type) + + def get_service_name(self, service_type): + key = '{service_type}_service_name'.format(service_type=service_type) + return self.config.get(key, service_type) From b631da86fee360533111eb9993e215d6cb64f522 Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Mon, 22 Jun 2015 05:40:40 +0000 Subject: [PATCH 102/365] Normalize project_name aliases We arent normalizing keys before we check for project_name aliases, therefore using hyphenated versions of the aliases fail. Change-Id: I3e0aa9dc38bbafc3c3a205f08b65abbd4528e874 --- os_client_config/config.py | 6 ++++-- os_client_config/tests/base.py | 8 ++++++++ os_client_config/tests/test_config.py | 7 +++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index bc8ab1e33..d2bc8166a 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -253,8 +253,10 @@ class OpenStackConfig(object): def _fix_backwards_project(self, cloud): # Do the lists backwards so that project_name is the ultimate winner mappings = { - 'project_name': ('tenant_id', 'project_id', - 'tenant_name', 'project_name'), + 'project_name': ('tenant_id', 'tenant-id', + 'project_id', 'project-id', + 'tenant_name', 'tenant-name', + 'project_name', 'project-name'), } for target_key, possible_values in mappings.items(): target = None diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 967f119dd..0a4f456aa 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -64,6 +64,14 @@ USER_CONF = { }, 'region_name': 'test-region', }, + '_test_cloud_hyphenated': { + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project-id': '12345', + }, + 'region_name': 'test-region', + } }, 'cache': {'max_age': 1}, } diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index a6ba2ee83..f23a6b987 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -87,6 +87,12 @@ class TestConfig(base.TestCase): cc = c.get_one_cloud('_test-cloud-int-project_') self.assertEqual('12345', cc.auth['project_name']) + def test_get_one_cloud_with_hyphenated_project_id(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test_cloud_hyphenated') + self.assertEqual('12345', cc.auth['project_name']) + def test_no_environ(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) @@ -112,6 +118,7 @@ class TestConfig(base.TestCase): self.assertEqual( ['_test-cloud-int-project_', '_test-cloud_', + '_test_cloud_hyphenated', '_test_cloud_no_vendor', ], sorted(c.get_cloud_names())) From 65dd84515be415fe9f0c5751ecb8b6c51e3fabcb Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 26 Jun 2015 10:58:37 -0400 Subject: [PATCH 103/365] Add support for indicating preference for IPv6 People, such as Infra, would like to use IPv6 when it's there, but don't want to need to write the "if ipv6, awesome, else, ipv4" code all the time. Change-Id: I870955863f1e8851c684dc604584c1ef3e20dd6b --- README.rst | 31 +++++++++++++++++++++ os_client_config/cloud_config.py | 7 ++++- os_client_config/config.py | 15 +++++++++- os_client_config/tests/base.py | 3 ++ os_client_config/tests/test_cloud_config.py | 8 ++++++ os_client_config/tests/test_config.py | 12 ++++++++ os_client_config/tests/test_environ.py | 8 ++++++ 7 files changed, 82 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 9cdda45dc..b265341a1 100644 --- a/README.rst +++ b/README.rst @@ -185,6 +185,37 @@ are connecting to OpenStack can share a cache should you desire. dns_service_type: hpext:dns +IPv6 +---- + +IPv6 may be a thing you would prefer to use not only if the cloud supports it, +but also if your local machine support it. A simple boolean flag is settable +either in an environment variable, `OS_PREFER_IPV6`, or in the client section +of the clouds.yaml. + +:: + client: + prefer_ipv6: true + clouds: + mordred: + profile: hp + auth: + username: mordred@inaugust.com + password: XXXXXXXXX + project_name: mordred@inaugust.com + region_name: region-b.geo-1 + monty: + profile: rax + auth: + username: mordred@inaugust.com + password: XXXXXXXXX + project_name: mordred@inaugust.com + region_name: DFW + +The above snippet will tell client programs to prefer returning an IPv6 +address. This will result in calls to, for instance, `shade`'s `get_public_ip` +to return an IPv4 address on HP, and an IPv6 address on Rackspace. + Usage ----- diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 19f1bb4ee..d0586f218 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -16,10 +16,11 @@ import warnings class CloudConfig(object): - def __init__(self, name, region, config): + def __init__(self, name, region, config, prefer_ipv6=False): self.name = name self.region = region self.config = config + self._prefer_ipv6 = prefer_ipv6 def __getattr__(self, key): """Return arbitrary attributes.""" @@ -96,3 +97,7 @@ class CloudConfig(object): def get_service_name(self, service_type): key = '{service_type}_service_name'.format(service_type=service_type) return self.config.get(key, service_type) + + @property + def prefer_ipv6(self): + return self._prefer_ipv6 diff --git a/os_client_config/config.py b/os_client_config/config.py index d2bc8166a..7961e72d2 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -68,6 +68,8 @@ def set_default(key, value): def get_boolean(value): + if type(value) is bool: + return value if value.lower() == 'true': return True return False @@ -116,6 +118,14 @@ class OpenStackConfig(object): if 'clouds' not in self.cloud_config: self.cloud_config['clouds'] = {} + # Grab ipv6 preference settings from env + client_config = self.cloud_config.get('client', {}) + self.prefer_ipv6 = get_boolean( + os.environ.pop( + 'OS_PREFER_IPV6', client_config.get( + 'prefer_ipv6', client_config.get( + 'prefer-ipv6', False)))) + # Next, process environment variables and add them to the mix self.envvar_key = os.environ.pop('OS_CLOUD_NAME', 'envvars') if self.envvar_key in self.cloud_config['clouds']: @@ -427,13 +437,16 @@ class OpenStackConfig(object): if hasattr(value, 'format'): config[key] = value.format(**config) + prefer_ipv6 = config.pop('prefer_ipv6', self.prefer_ipv6) + if cloud is None: cloud_name = '' else: cloud_name = str(cloud) return cloud_config.CloudConfig( name=cloud_name, region=config['region_name'], - config=self._normalize_keys(config)) + config=self._normalize_keys(config), + prefer_ipv6=prefer_ipv6) @staticmethod def set_one_cloud(config_file, cloud, set_config=None): diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 0a4f456aa..8bb9145d2 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -38,6 +38,9 @@ VENDOR_CONF = { } } USER_CONF = { + 'client': { + 'prefer_ipv6': True, + }, 'clouds': { '_test-cloud_': { 'profile': '_test_cloud_in_our_cloud', diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 5f964dbeb..1a20ebf96 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -39,6 +39,9 @@ class TestCloudConfig(base.TestCase): # Lookup mystery attribute self.assertIsNone(cc.x) + # Test default ipv6 + self.assertFalse(cc.prefer_ipv6) + def test_iteration(self): cc = cloud_config.CloudConfig("test1", "region-al", fake_config_dict) self.assertTrue('a' in cc) @@ -100,3 +103,8 @@ class TestCloudConfig(base.TestCase): cc = cloud_config.CloudConfig("test1", "region-xx", config_dict) (verify, cert) = cc.get_requests_verify_args() self.assertEqual(("cert", "key"), cert) + + def test_ipv6(self): + cc = cloud_config.CloudConfig( + "test1", "region-al", fake_config_dict, prefer_ipv6=True) + self.assertTrue(cc.prefer_ipv6) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index f23a6b987..4c244d2a4 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -107,6 +107,18 @@ class TestConfig(base.TestCase): self.useFixture(fixtures.EnvironmentVariable(k)) c.get_one_cloud(cloud='defaults') + def test_prefer_ipv6_true(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud(cloud='_test-cloud_') + self.assertTrue(cc.prefer_ipv6) + + def test_prefer_ipv6_false(self): + c = config.OpenStackConfig(config_files=[self.no_yaml], + vendor_files=[self.no_yaml]) + cc = c.get_one_cloud(cloud='defaults') + self.assertFalse(cc.prefer_ipv6) + def test_get_one_cloud_auth_merge(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml]) cc = c.get_one_cloud(cloud='_test-cloud_', auth={'username': 'user'}) diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 365ad6780..ff44afc32 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -51,6 +51,14 @@ class TestEnviron(base.TestCase): cc = c.get_one_cloud('override') self._assert_cloud_details(cc) + def test_envvar_prefer_ipv6_override(self): + self.useFixture( + fixtures.EnvironmentVariable('OS_PREFER_IPV6', 'false')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test-cloud_') + self.assertFalse(cc.prefer_ipv6) + def test_environ_exists(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From 50f05448982b5fafd9d9a7783b639dd145090a0d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 6 Jun 2015 09:40:03 -0400 Subject: [PATCH 104/365] Clean up vendor data There are some clear central defaults. Call them out and don't repeat them. Also, ran the yaml files through a flamel conversion so they're all consistently formatted. Change-Id: Id19116c5e8266c109cf015d097cb6cb35f1beae8 --- os_client_config/config.py | 9 ++--- os_client_config/defaults.py | 44 +++++++++++------------ os_client_config/defaults.yaml | 16 +++++++++ os_client_config/vendors/__init__.py | 17 ++++++--- os_client_config/vendors/auro.yaml | 7 ++-- os_client_config/vendors/dreamhost.yaml | 6 ++-- os_client_config/vendors/hp.yaml | 8 ++--- os_client_config/vendors/ovh.yaml | 8 ++--- os_client_config/vendors/rackspace.yaml | 15 ++++---- os_client_config/vendors/runabove.yaml | 9 ++--- os_client_config/vendors/unitedstack.yaml | 9 ++--- os_client_config/vendors/vexxhost.yaml | 7 ++-- 12 files changed, 86 insertions(+), 69 deletions(-) create mode 100644 os_client_config/defaults.yaml diff --git a/os_client_config/config.py b/os_client_config/config.py index d2bc8166a..9eb13b3ba 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -218,7 +218,7 @@ class OpenStackConfig(object): # Expand a profile if it exists. 'cloud' is an old confusing name # for this. profile_name = our_cloud.get('profile', our_cloud.get('cloud', None)) - if profile_name: + if profile_name and profile_name != self.envvar_key: if 'cloud' in our_cloud: warnings.warn( "{0} use the keyword 'cloud' to reference a known " @@ -228,9 +228,10 @@ class OpenStackConfig(object): if vendor_file and profile_name in vendor_file['public-clouds']: _auth_update(cloud, vendor_file['public-clouds'][profile_name]) else: - try: - _auth_update(cloud, vendors.CLOUD_DEFAULTS[profile_name]) - except KeyError: + profile_data = vendors.get_profile(profile_name) + if profile_data: + _auth_update(cloud, profile_data) + else: # Can't find the requested vendor config, go about business warnings.warn("Couldn't find the vendor profile '{0}', for" " the cloud '{1}'".format(profile_name, diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index 8bf693d8d..a274767bd 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -12,29 +12,29 @@ # License for the specific language governing permissions and limitations # under the License. -_defaults = dict( - api_timeout=None, - auth_type='password', - baremetal_api_version='1', - compute_api_version='2', - database_api_version='1.0', - endpoint_type='public', - floating_ip_source='neutron', - identity_api_version='2', - image_api_use_tasks=False, - image_api_version='1', - network_api_version='2', - object_api_version='1', - secgroup_source='neutron', - volume_api_version='1', - disable_vendor_agent={}, - # SSL Related args - verify=True, - cacert=None, - cert=None, - key=None, -) +import os + +import yaml + +_yaml_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'defaults.yaml') +_defaults = None def get_defaults(): + global _defaults + if not _defaults: + # Python language specific defaults + # These are defaults related to use of python libraries, they are + # not qualities of a cloud. + _defaults = dict( + api_timeout=None, + verify=True, + cacert=None, + cert=None, + key=None, + ) + with open(_yaml_path, 'r') as yaml_file: + _defaults.update(yaml.load(yaml_file.read())) + return _defaults.copy() diff --git a/os_client_config/defaults.yaml b/os_client_config/defaults.yaml new file mode 100644 index 000000000..e4c900062 --- /dev/null +++ b/os_client_config/defaults.yaml @@ -0,0 +1,16 @@ +auth_type: password +baremetal_api_version: '1' +compute_api_version: '2' +database_api_version: '1.0' +disable_vendor_agent: {} +dns_api_version: '2' +endpoint_type: public +floating_ip_source: neutron +identity_api_version: '2' +image_api_use_tasks: false +image_api_version: '1' +image_format: qcow2 +network_api_version: '2' +object_api_version: '1' +secgroup_source: neutron +volume_api_version: '1' diff --git a/os_client_config/vendors/__init__.py b/os_client_config/vendors/__init__.py index 0ccb04c5d..367e3182b 100644 --- a/os_client_config/vendors/__init__.py +++ b/os_client_config/vendors/__init__.py @@ -17,9 +17,16 @@ import os import yaml -vendors_path = os.path.dirname(os.path.realpath(__file__)) +_vendors_path = os.path.dirname(os.path.realpath(__file__)) +_vendor_defaults = None -CLOUD_DEFAULTS = {} -for vendor in glob.glob(os.path.join(vendors_path, '*.yaml')): - with open(vendor, 'r') as f: - CLOUD_DEFAULTS.update(yaml.safe_load(f)) + +def get_profile(profile_name): + global _vendor_defaults + if _vendor_defaults is None: + _vendor_defaults = {} + for vendor in glob.glob(os.path.join(_vendors_path, '*.yaml')): + with open(vendor, 'r') as f: + vendor_data = yaml.load(f) + _vendor_defaults[vendor_data['name']] = vendor_data['profile'] + return _vendor_defaults.get(profile_name) diff --git a/os_client_config/vendors/auro.yaml b/os_client_config/vendors/auro.yaml index da1491dec..987838bbb 100644 --- a/os_client_config/vendors/auro.yaml +++ b/os_client_config/vendors/auro.yaml @@ -1,10 +1,7 @@ ---- -auro: +name: auro +profile: auth: auth_url: https://api.auro.io:5000/v2.0 region_name: RegionOne - identity_api_version: 2 - image_api_version: 1 - image_format: qcow2 secgroup_source: nova floating_ip_source: nova diff --git a/os_client_config/vendors/dreamhost.yaml b/os_client_config/vendors/dreamhost.yaml index 63f15c31d..5e10d1403 100644 --- a/os_client_config/vendors/dreamhost.yaml +++ b/os_client_config/vendors/dreamhost.yaml @@ -1,7 +1,7 @@ ---- -dreamhost: +name: dreamhost +profile: auth: auth_url: https://keystone.dream.io/v2.0 region_name: RegionOne - image_api_version: 2 + image_api_version: '2' image_format: raw diff --git a/os_client_config/vendors/hp.yaml b/os_client_config/vendors/hp.yaml index 2ac551781..277343325 100644 --- a/os_client_config/vendors/hp.yaml +++ b/os_client_config/vendors/hp.yaml @@ -1,8 +1,6 @@ ---- -hp: +name: hp +profile: auth: auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 - region_name: region-b.geo-1 + region_name: region-a.geo-1,region-b.geo-1 dns_service_type: hpext:dns - image_api_version: 1 - image_format: qcow2 diff --git a/os_client_config/vendors/ovh.yaml b/os_client_config/vendors/ovh.yaml index 5b4426f18..f83437297 100644 --- a/os_client_config/vendors/ovh.yaml +++ b/os_client_config/vendors/ovh.yaml @@ -1,10 +1,6 @@ ---- -ovh: +name: ovh +profile: auth: auth_url: https://auth.cloud.ovh.net/v2.0 region_name: SBG1 - identity_api_version: 2 - image_api_version: 1 image_format: raw - secgroup_source: neutron - floating_ip_source: neutron diff --git a/os_client_config/vendors/rackspace.yaml b/os_client_config/vendors/rackspace.yaml index 7af3cab96..a350584c4 100644 --- a/os_client_config/vendors/rackspace.yaml +++ b/os_client_config/vendors/rackspace.yaml @@ -1,14 +1,15 @@ ---- -rackspace: +name: rackspace +profile: auth: - auth_url: https://identity.api.rackspacecloud.com/v2.0 - database_service_type: 'rax:database' + auth_url: https://identity.api.rackspacecloud.com/v2.0/ + region_name: DFW,ORD,IAD,SYD,HKG + database_service_type: rax:database compute_service_name: cloudServersOpenStack - image_api_version: 2 - image_api_use_tasks: True + image_api_version: '2' + image_api_use_tasks: true image_format: vhd floating_ip_source: None secgroup_source: None disable_vendor_agent: vm_mode: hvm - xenapi_use_agent: False + xenapi_use_agent: false diff --git a/os_client_config/vendors/runabove.yaml b/os_client_config/vendors/runabove.yaml index c9641dd41..381020ab8 100644 --- a/os_client_config/vendors/runabove.yaml +++ b/os_client_config/vendors/runabove.yaml @@ -1,7 +1,8 @@ ---- -runabove: +name: runabove +profile: auth: auth_url: https://auth.runabove.io/v2.0 - image_api_version: 2 + region_name: SBG-1,BHS-1 + image_api_version: '2' image_format: qcow2 - floating_ip_sourc: None + floating_ip_source: None diff --git a/os_client_config/vendors/unitedstack.yaml b/os_client_config/vendors/unitedstack.yaml index 43c600a6f..db9b61239 100644 --- a/os_client_config/vendors/unitedstack.yaml +++ b/os_client_config/vendors/unitedstack.yaml @@ -1,8 +1,9 @@ ---- -unitedstack: +name: unitedstack +profile: auth: auth_url: https://identity.api.ustack.com/v3 - identity_api_version: 3 - image_api_version: 2 + region_name: bj1,gd1 + identity_api_version: '3' + image_api_version: '2' image_format: raw floating_ip_source: None diff --git a/os_client_config/vendors/vexxhost.yaml b/os_client_config/vendors/vexxhost.yaml index 5247ef436..f67c644c6 100644 --- a/os_client_config/vendors/vexxhost.yaml +++ b/os_client_config/vendors/vexxhost.yaml @@ -1,8 +1,7 @@ ---- -vexxhost: +name: vexxhost +profile: auth: auth_url: http://auth.api.thenebulacloud.com:5000/v2.0/ region_name: ca-ymq-1 - image_api_version: 2 - image_format: qcow2 + image_api_version: '2' floating_ip_source: None From 9d3cc7969b7022ac850d98321fa038a09a70f939 Mon Sep 17 00:00:00 2001 From: Spencer Krum Date: Wed, 8 Jul 2015 10:25:51 -0700 Subject: [PATCH 105/365] Fix rendering issue in Readme Change-Id: If089b0331c6b40e983d81623ee3a6a541f93a45a --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index b265341a1..e2f75cd70 100644 --- a/README.rst +++ b/README.rst @@ -194,6 +194,7 @@ either in an environment variable, `OS_PREFER_IPV6`, or in the client section of the clouds.yaml. :: + client: prefer_ipv6: true clouds: From 1065ea4dbf91a1416e267223d0db96719490653b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 4 Jul 2015 10:55:25 -0400 Subject: [PATCH 106/365] Add support for configuring region lists with yaml yaml supports encoding lists in a manner that is cleaner than comma separated lists. While the old commas-in-region_name will still work, add a 'regions' option that can be used to configure lists of regions. Remove the comma-separated-list from the docs so that people don't try to use it - even though it will work. Change-Id: Ieb0aedb9c03fd26e644e9ba733b935f2c69daaf0 --- README.rst | 7 ++++-- os_client_config/config.py | 24 +++++++++++++++------ os_client_config/tests/base.py | 11 ++++++++++ os_client_config/tests/test_config.py | 26 ++++++++++++++++++++++- os_client_config/vendors/hp.yaml | 4 +++- os_client_config/vendors/rackspace.yaml | 7 +++++- os_client_config/vendors/runabove.yaml | 4 +++- os_client_config/vendors/unitedstack.yaml | 4 +++- 8 files changed, 73 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index b265341a1..19927135c 100644 --- a/README.rst +++ b/README.rst @@ -105,7 +105,10 @@ An example config file is probably helpful: username: openstackci password: XXXXXXXX project_id: 610275 - region_name: DFW,ORD,IAD + regions: + - DFW + - ORD + - IAD You may note a few things. First, since `auth_url` settings are silly and embarrasingly ugly, known cloud vendor profile information is included and @@ -116,7 +119,7 @@ knows that so that you don't have to. In case the cloud vendor profile is not available, you can provide one called `clouds-public.yaml`, following the same location rules previously mentioned for the config files. -Also, `region_name` can be a list of regions. When you call `get_all_clouds`, +`regions` can be a list of regions. When you call `get_all_clouds`, you'll get a cloud config object for each cloud/region combo. As seen with `dns_service_type`, any setting that makes sense to be per-service, diff --git a/os_client_config/config.py b/os_client_config/config.py index 55328cc0e..93d7aa0b8 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -199,14 +199,24 @@ class OpenStackConfig(object): return self._cache_arguments def _get_regions(self, cloud): - try: - return self.cloud_config['clouds'][cloud]['region_name'] - except KeyError: - # No region configured - return '' + if cloud not in self.cloud_config['clouds']: + return [''] + config = self._normalize_keys(self.cloud_config['clouds'][cloud]) + if 'regions' in config: + return config['regions'] + elif 'region_name' in config: + regions = config['region_name'].split(',') + if len(regions) > 1: + warnings.warn( + "Comma separated lists in region_name are deprecated." + " Please use a yaml list in the regions" + " parameter in {0} instead.".format(self.config_filename)) + return regions + else: + return [''] def _get_region(self, cloud=None): - return self._get_regions(cloud).split(',')[0] + return self._get_regions(cloud)[0] def get_cloud_names(self): return self.cloud_config['clouds'].keys() @@ -301,7 +311,7 @@ class OpenStackConfig(object): clouds = [] for cloud in self.get_cloud_names(): - for region in self._get_regions(cloud).split(','): + for region in self._get_regions(cloud): clouds.append(self.get_one_cloud(cloud, region_name=region)) return clouds diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 8bb9145d2..569b52ea7 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -67,6 +67,17 @@ USER_CONF = { }, 'region_name': 'test-region', }, + '_test_cloud_regions': { + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project-id': 'testproject', + }, + 'regions': [ + 'region1', + 'region2', + ], + }, '_test_cloud_hyphenated': { 'auth': { 'username': 'testuser', diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 4c244d2a4..dde7d83c7 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -32,7 +32,11 @@ class TestConfig(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) clouds = c.get_all_clouds() - user_clouds = [cloud for cloud in base.USER_CONF['clouds'].keys()] + # We add one by hand because the regions cloud is going to exist + # twice since it has two regions in it + user_clouds = [ + cloud for cloud in base.USER_CONF['clouds'].keys() + ] + ['_test_cloud_regions'] configured_clouds = [cloud.name for cloud in clouds] self.assertItemsEqual(user_clouds, configured_clouds) @@ -132,6 +136,7 @@ class TestConfig(base.TestCase): '_test-cloud_', '_test_cloud_hyphenated', '_test_cloud_no_vendor', + '_test_cloud_regions', ], sorted(c.get_cloud_names())) c = config.OpenStackConfig(config_files=[self.no_yaml], @@ -216,6 +221,25 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(cc.region_name, 'test-region') self.assertIsNone(cc.snack_type) + def test_get_one_cloud_no_argparse_regions(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud(cloud='_test_cloud_regions', argparse=None) + self._assert_cloud_details(cc) + self.assertEqual(cc.region_name, 'region1') + self.assertIsNone(cc.snack_type) + + def test_get_one_cloud_no_argparse_region2(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud( + cloud='_test_cloud_regions', region_name='region2', argparse=None) + self._assert_cloud_details(cc) + self.assertEqual(cc.region_name, 'region2') + self.assertIsNone(cc.snack_type) + def test_fix_env_args(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) diff --git a/os_client_config/vendors/hp.yaml b/os_client_config/vendors/hp.yaml index 277343325..4da49568d 100644 --- a/os_client_config/vendors/hp.yaml +++ b/os_client_config/vendors/hp.yaml @@ -2,5 +2,7 @@ name: hp profile: auth: auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 - region_name: region-a.geo-1,region-b.geo-1 + regions: + - region-a.geo-1 + - region-b.geo-1 dns_service_type: hpext:dns diff --git a/os_client_config/vendors/rackspace.yaml b/os_client_config/vendors/rackspace.yaml index a350584c4..ad7825dec 100644 --- a/os_client_config/vendors/rackspace.yaml +++ b/os_client_config/vendors/rackspace.yaml @@ -2,7 +2,12 @@ name: rackspace profile: auth: auth_url: https://identity.api.rackspacecloud.com/v2.0/ - region_name: DFW,ORD,IAD,SYD,HKG + regions: + - DFW + - HKG + - IAD + - ORD + - SYD database_service_type: rax:database compute_service_name: cloudServersOpenStack image_api_version: '2' diff --git a/os_client_config/vendors/runabove.yaml b/os_client_config/vendors/runabove.yaml index 381020ab8..57e75f37a 100644 --- a/os_client_config/vendors/runabove.yaml +++ b/os_client_config/vendors/runabove.yaml @@ -2,7 +2,9 @@ name: runabove profile: auth: auth_url: https://auth.runabove.io/v2.0 - region_name: SBG-1,BHS-1 + regions: + - BHS-1 + - SBG-1 image_api_version: '2' image_format: qcow2 floating_ip_source: None diff --git a/os_client_config/vendors/unitedstack.yaml b/os_client_config/vendors/unitedstack.yaml index db9b61239..f22bd8abe 100644 --- a/os_client_config/vendors/unitedstack.yaml +++ b/os_client_config/vendors/unitedstack.yaml @@ -2,7 +2,9 @@ name: unitedstack profile: auth: auth_url: https://identity.api.ustack.com/v3 - region_name: bj1,gd1 + regions: + - bj1 + - gd1 identity_api_version: '3' image_api_version: '2' image_format: raw From 6523cf62fa369fe1909a951ac7bce465aa222c06 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 4 Jul 2015 11:03:07 -0400 Subject: [PATCH 107/365] Specify the config file with environment variable The fine folks at ansible want to be able to specify a specific location for the config file with an env var which seems like a perfectly reasonable thing to allow. Inject the specified file at the beginning of the list so that it'll be the first one found. Change-Id: Ib1947be1c0ae812e9cb83c7b99168c05dfc6fa6a Co-authored-by: Chris Church --- README.rst | 4 ++++ os_client_config/config.py | 4 ++++ os_client_config/tests/test_environ.py | 9 +++++++++ 3 files changed, 17 insertions(+) diff --git a/README.rst b/README.rst index 19927135c..13e6f2a51 100644 --- a/README.rst +++ b/README.rst @@ -43,6 +43,10 @@ locations: The first file found wins. +You can also set the environment variable `OS_CLIENT_CONFIG_FILE` to an +absolute path of a file to look for and that location will be inserted at the +front of the file search list. + The keys are all of the keys you'd expect from `OS_*` - except lower case and without the OS prefix. So, region name is set with `region_name`. diff --git a/os_client_config/config.py b/os_client_config/config.py index 93d7aa0b8..203772e21 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -106,6 +106,10 @@ class OpenStackConfig(object): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES + config_file_override = os.environ.pop('OS_CLIENT_CONFIG_FILE', None) + if config_file_override: + self._config_files.insert(0, config_file_override) + self.defaults = defaults.get_defaults() if override_defaults: self.defaults.update(override_defaults) diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index ff44afc32..7f284c5eb 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -84,3 +84,12 @@ class TestEnviron(base.TestCase): self._assert_cloud_details(cc) cc = c.get_one_cloud('_test_cloud_no_vendor') self._assert_cloud_details(cc) + + def test_config_file_override(self): + self.useFixture( + fixtures.EnvironmentVariable( + 'OS_CLIENT_CONFIG_FILE', self.cloud_yaml)) + c = config.OpenStackConfig(config_files=[], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test-cloud_') + self._assert_cloud_details(cc) From b74df460a821d522d78ee76ba699bc4265efa2bc Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Tue, 14 Jul 2015 10:54:21 -0400 Subject: [PATCH 108/365] Fix set_default() when used before config init Using set_default() before initializing OpenStackConfig would cause an error since the _defaults dict would still be None and not an actual dict. This corrects that by calling get_defaults() to make sure it is initialized properly, and also adds a warning to note that the method is now deprecated. Change-Id: I81803c680b614f9bee47c6f69a4efffa638dcebc --- os_client_config/config.py | 5 +++++ os_client_config/tests/test_config.py | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 203772e21..89503e752 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -64,6 +64,11 @@ BOOL_KEYS = ('insecure', 'cache') # Remove this sometime in June 2015 once OSC is comfortably # changed-over and global-defaults is updated. def set_default(key, value): + warnings.warn( + "Use of set_default() is deprecated. Defaults should be set with the " + "`override_defaults` parameter of OpenStackConfig." + ) + defaults.get_defaults() # make sure the dict is initialized defaults._defaults[key] = value diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index dde7d83c7..1444ed557 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -252,9 +252,26 @@ class TestConfigArgparse(base.TestCase): class TestConfigDefault(base.TestCase): + def setUp(self): + super(TestConfigDefault, self).setUp() + + # Reset defaults after each test so that other tests are + # not affected by any changes. + self.addCleanup(self._reset_defaults) + + def _reset_defaults(self): + defaults._defaults = None + def test_set_no_default(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud(cloud='_test-cloud_', argparse=None) self._assert_cloud_details(cc) - self.assertEqual(cc.auth_type, 'password') + self.assertEqual('password', cc.auth_type) + + def test_set_default_before_init(self): + config.set_default('auth_type', 'token') + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud(cloud='_test-cloud_', argparse=None) + self.assertEqual('token', cc.auth_type) From 3891816acb6dd287f89498dd0fcb714b722f9252 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 16 Jul 2015 09:09:35 -0400 Subject: [PATCH 109/365] Remove region list from single cloud regions is a list of regions that can be used to create more than one CloudConfig object for each cloud/region combo. The regions field itself makes no sense to set on each of those CloudConfigs, so do not pass it through. Change-Id: I76b3bb3bc4778223d72f86d01d02ce150651b3b9 --- os_client_config/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 93d7aa0b8..6c3abff50 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -423,6 +423,10 @@ class OpenStackConfig(object): config = self._get_base_cloud_config(cloud) + # Regions is a list that we can use to create a list of cloud/region + # objects. It does not belong in the single-cloud dict + config.pop('regions', None) + # Can't just do update, because None values take over for (key, val) in iter(args.items()): if val is not None: From b4145438fd603aca96f196383f3aee8b0973d24c Mon Sep 17 00:00:00 2001 From: TerryHowe Date: Mon, 13 Jul 2015 13:51:11 -0600 Subject: [PATCH 110/365] Have service name default to None It seems like having service name default to service_type would not work in a lot of situations. Change-Id: Ia70242fad346c1681fa4abca9d604aea3ae002dd --- os_client_config/cloud_config.py | 2 +- os_client_config/tests/test_cloud_config.py | 29 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index d0586f218..5ac061208 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -96,7 +96,7 @@ class CloudConfig(object): def get_service_name(self, service_type): key = '{service_type}_service_name'.format(service_type=service_type) - return self.config.get(key, service_type) + return self.config.get(key, None) @property def prefer_ipv6(self): diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 1a20ebf96..2af568fbc 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -17,6 +17,15 @@ from os_client_config.tests import base fake_config_dict = {'a': 1, 'os_b': 2, 'c': 3, 'os_c': 4} +fake_services_dict = { + 'compute_api_version': 2, + 'compute_region_name': 'region-bl', + 'endpoint_type': 'public', + 'image_service_type': 'mage', + 'identity_endpoint_type': 'admin', + 'identity_service_name': 'locks', + 'auth': {'password': 'hunter2', 'username': 'AzureDiamond'}, +} class TestCloudConfig(base.TestCase): @@ -108,3 +117,23 @@ class TestCloudConfig(base.TestCase): cc = cloud_config.CloudConfig( "test1", "region-al", fake_config_dict, prefer_ipv6=True) self.assertTrue(cc.prefer_ipv6) + + def test_getters(self): + cc = cloud_config.CloudConfig("test1", "region-al", fake_services_dict) + + self.assertEqual(['compute', 'identity', 'image'], + sorted(cc.get_services())) + self.assertEqual({'password': 'hunter2', 'username': 'AzureDiamond'}, + cc.get_auth_args()) + self.assertEqual('public', cc.get_endpoint_type()) + self.assertEqual('public', cc.get_endpoint_type('image')) + self.assertEqual('admin', cc.get_endpoint_type('identity')) + self.assertEqual('region-al', cc.get_region_name()) + self.assertEqual('region-al', cc.get_region_name('image')) + self.assertEqual('region-bl', cc.get_region_name('compute')) + self.assertEqual(None, cc.get_api_version('image')) + self.assertEqual(2, cc.get_api_version('compute')) + self.assertEqual('mage', cc.get_service_type('image')) + self.assertEqual('compute', cc.get_service_type('compute')) + self.assertEqual(None, cc.get_service_name('compute')) + self.assertEqual('locks', cc.get_service_name('identity')) From 0b698134db7fef262304f1eb55b374dc36ce93b1 Mon Sep 17 00:00:00 2001 From: TerryHowe Date: Tue, 14 Jul 2015 12:30:16 -0600 Subject: [PATCH 111/365] Rename 'endpoint_type' to 'interface' The keystone folks seem to feel that interface is a better name than endpoint type. Change-Id: Ibfc54e725b6dae843c07f7786f51f9fb9141c983 --- os_client_config/cloud_config.py | 9 +++++---- os_client_config/config.py | 8 ++++++++ os_client_config/defaults.yaml | 2 +- os_client_config/tests/test_cloud_config.py | 10 +++++----- os_client_config/tests/test_config.py | 21 +++++++++++++++++++++ 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 5ac061208..fd35aeacd 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -74,11 +74,12 @@ class CloudConfig(object): def get_auth_args(self): return self.config['auth'] - def get_endpoint_type(self, service_type=None): + def get_interface(self, service_type=None): + interface = self.config.get('interface') if not service_type: - return self.config['endpoint_type'] - key = '{service_type}_endpoint_type'.format(service_type=service_type) - return self.config.get(key, self.config['endpoint_type']) + return interface + key = '{service_type}_interface'.format(service_type=service_type) + return self.config.get(key, interface) def get_region_name(self, service_type=None): if not service_type: diff --git a/os_client_config/config.py b/os_client_config/config.py index 89503e752..25fbf64d4 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -278,6 +278,7 @@ class OpenStackConfig(object): def _fix_backwards_madness(self, cloud): cloud = self._fix_backwards_project(cloud) cloud = self._fix_backwards_auth_plugin(cloud) + cloud = self._fix_backwards_interface(cloud) return cloud def _fix_backwards_project(self, cloud): @@ -315,6 +316,13 @@ class OpenStackConfig(object): cloud[target_key] = target return cloud + def _fix_backwards_interface(self, cloud): + for key in cloud.keys(): + if key.endswith('endpoint_type'): + target_key = key.replace('endpoint_type', 'interface') + cloud[target_key] = cloud.pop(key) + return cloud + def get_all_clouds(self): clouds = [] diff --git a/os_client_config/defaults.yaml b/os_client_config/defaults.yaml index e4c900062..9130228f2 100644 --- a/os_client_config/defaults.yaml +++ b/os_client_config/defaults.yaml @@ -4,7 +4,7 @@ compute_api_version: '2' database_api_version: '1.0' disable_vendor_agent: {} dns_api_version: '2' -endpoint_type: public +interface: public floating_ip_source: neutron identity_api_version: '2' image_api_use_tasks: false diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 2af568fbc..29cbba037 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -20,9 +20,9 @@ fake_config_dict = {'a': 1, 'os_b': 2, 'c': 3, 'os_c': 4} fake_services_dict = { 'compute_api_version': 2, 'compute_region_name': 'region-bl', - 'endpoint_type': 'public', + 'interface': 'public', 'image_service_type': 'mage', - 'identity_endpoint_type': 'admin', + 'identity_interface': 'admin', 'identity_service_name': 'locks', 'auth': {'password': 'hunter2', 'username': 'AzureDiamond'}, } @@ -125,9 +125,9 @@ class TestCloudConfig(base.TestCase): sorted(cc.get_services())) self.assertEqual({'password': 'hunter2', 'username': 'AzureDiamond'}, cc.get_auth_args()) - self.assertEqual('public', cc.get_endpoint_type()) - self.assertEqual('public', cc.get_endpoint_type('image')) - self.assertEqual('admin', cc.get_endpoint_type('identity')) + self.assertEqual('public', cc.get_interface()) + self.assertEqual('public', cc.get_interface('compute')) + self.assertEqual('admin', cc.get_interface('identity')) self.assertEqual('region-al', cc.get_region_name()) self.assertEqual('region-al', cc.get_region_name('image')) self.assertEqual('region-bl', cc.get_region_name('compute')) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 1444ed557..6bb65fce5 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -275,3 +275,24 @@ class TestConfigDefault(base.TestCase): vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud(cloud='_test-cloud_', argparse=None) self.assertEqual('token', cc.auth_type) + + +class TestBackwardsCompatibility(base.TestCase): + + def test_set_no_default(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cloud = { + 'identity_endpoint_type': 'admin', + 'compute_endpoint_type': 'private', + 'endpoint_type': 'public', + 'auth_type': 'v3password', + } + result = c._fix_backwards_interface(cloud) + expected = { + 'identity_interface': 'admin', + 'compute_interface': 'private', + 'interface': 'public', + 'auth_type': 'v3password', + } + self.assertEqual(expected, result) From b292db766ff49a9d34a9b8877440add6d0253335 Mon Sep 17 00:00:00 2001 From: TerryHowe Date: Thu, 16 Jul 2015 14:52:24 -0600 Subject: [PATCH 112/365] Remove py26 and py33 from tox.ini Change-Id: I79a9dceffc0432398bda70949db2e523cbbff515 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 860e3378b..af6ede141 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py33,py34,py26,py27,pypy,pep8 +envlist = py34,py27,pypy,pep8 skipsdist = True [testenv] From d6d2cbe5e1ada67f78e7dd1bedebe8a57fcac8fe Mon Sep 17 00:00:00 2001 From: TerryHowe Date: Fri, 17 Jul 2015 11:26:03 -0600 Subject: [PATCH 113/365] Remove requirements.txt from tox.ini From lifeless: pbr reflects the package dependencies from requirements.txt into the sdist that tox builds. Change-Id: Iaa6026a504cc53784aad5731e9afe8b684b3ede5 --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index af6ede141..7a2d3a07a 100644 --- a/tox.ini +++ b/tox.ini @@ -8,8 +8,7 @@ usedevelop = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt +deps = -r{toxinidir}/test-requirements.txt commands = python setup.py testr --slowest --testr-args='{posargs}' [testenv:pep8] From b2d10751eaea6e1b9fe15a202a18a98d2c25c95f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 27 Jul 2015 11:48:26 -0400 Subject: [PATCH 114/365] Add per-service endpoint overrides Make it possible to override a service's endpoint like service_type and service_name. This is already documented as working. Change-Id: I8764ed68f8a38563c4242d4b50e2158e99ed4109 --- os_client_config/cloud_config.py | 4 ++++ os_client_config/tests/test_cloud_config.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index fd35aeacd..f60303b2a 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -99,6 +99,10 @@ class CloudConfig(object): key = '{service_type}_service_name'.format(service_type=service_type) return self.config.get(key, None) + def get_endpoint(self, service_type): + key = '{service_type}_endpoint'.format(service_type=service_type) + return self.config.get(key, None) + @property def prefer_ipv6(self): return self._prefer_ipv6 diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 29cbba037..4f5260fc1 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -19,6 +19,7 @@ from os_client_config.tests import base fake_config_dict = {'a': 1, 'os_b': 2, 'c': 3, 'os_c': 4} fake_services_dict = { 'compute_api_version': 2, + 'compute_endpoint': 'http://compute.example.com', 'compute_region_name': 'region-bl', 'interface': 'public', 'image_service_type': 'mage', @@ -135,5 +136,9 @@ class TestCloudConfig(base.TestCase): self.assertEqual(2, cc.get_api_version('compute')) self.assertEqual('mage', cc.get_service_type('image')) self.assertEqual('compute', cc.get_service_type('compute')) + self.assertEqual('http://compute.example.com', + cc.get_endpoint('compute')) + self.assertEqual(None, + cc.get_endpoint('image')) self.assertEqual(None, cc.get_service_name('compute')) self.assertEqual('locks', cc.get_service_name('identity')) From 212ce988609fafc830fcb59f42d31eacefc62565 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 27 Jul 2015 12:15:46 -0400 Subject: [PATCH 115/365] Clarify floating ip use for vendors HP was not listed as needing a floating ip. Also, the non-floating IP case mentions direct routing, so amend the floating IP case to mention NAT for completeness and clarity. Change-Id: I220c41f6c822b7b6ffb1ec11038749153ef5a6ee --- doc/source/vendor-support.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 314e36229..3cf39e03f 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -37,6 +37,7 @@ region-b.geo-1 US East ============== ================ * DNS Service Type is `hpext:dns` +* Public IPv4 is provided via NAT with Neutron Floating IP rackspace --------- @@ -79,7 +80,7 @@ RegionOne Region One * Image API Version is 2 * Images must be in `raw` format -* Public IPv4 is provided via Floating IP from Neutron +* Public IPv4 is provided via NAT with Neutron Floating IP * IPv6 is provided to every server vexxhost @@ -139,7 +140,7 @@ RegionOne RegionOne ============== ================ * Identity API Version is 2 -* Public IPv4 is provided via Floating IP from Nova +* Public IPv4 is provided via NAT with Nova Floating IP * Floating IPs are provided by Nova * Security groups are provided by Nova From a8e8c809eeec0f09b5d9207c29efb3c6a1f9c088 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 1 Aug 2015 12:45:16 +1000 Subject: [PATCH 116/365] Align to generic password auth-type v2password can't handle v3 parameter names. But we align to those for sanity. Ensure that a user gets the v2plugin that can handle the auth instead of the one that can't. Change-Id: Ie693e613fd5d0e20a4837923300502b1de02364b --- os_client_config/config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 015a783d6..1840d18dc 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -314,6 +314,12 @@ class OpenStackConfig(object): target = cloud[key] del cloud[key] cloud[target_key] = target + # Because we force alignment to v3 nouns, we want to force + # use of the auth plugin that can do auto-selection and dealing + # with that based on auth parameters. v2password is basically + # completely broken + if cloud['auth_type'] == 'v2password': + cloud['auth_type'] = 'password' return cloud def _fix_backwards_interface(self, cloud): From dfef01a7cbb9d68ad443d182e548ec58143a3a1f Mon Sep 17 00:00:00 2001 From: Davide Guerri Date: Tue, 11 Aug 2015 23:54:39 +0100 Subject: [PATCH 117/365] Use the correct auth_plugin for token authentication Use ksc_auth.token_endpoint.Token as Keystone authentication plugin if auth_type is token_endpoint. Change-Id: I1a6d6dfe731527d7040cfbc2404fd5cf86ba5893 --- os_client_config/config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1840d18dc..240220ae9 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -379,8 +379,12 @@ class OpenStackConfig(object): def _validate_auth(self, config): # May throw a keystoneclient.exceptions.NoMatchingPlugin - plugin_options = ksc_auth.get_plugin_class( - config['auth_type']).get_options() + if config['auth_type'] == 'token_endpoint': + auth_plugin = ksc_auth.token_endpoint.Token + else: + auth_plugin = ksc_auth.get_plugin_class(config['auth_type']) + + plugin_options = auth_plugin.get_options() for p_opt in plugin_options: # if it's in config.auth, win, kill it from config dict From 13a04f76b9f13ec89a9d0aa8a7e9b064e8fa98b4 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Wed, 12 Aug 2015 19:56:51 +0000 Subject: [PATCH 118/365] Ignore infra CI env vars When running tests in the infra CI system, some OS_ environment variables are defined (e.g., OS_TEST_TIMEOUT, OS_TEST_PATH, OS_STDERR_CAPTURE, OS_STDOUT_CAPTURE) and this makes o-c-c think that we are using environment variables for an envvar cloud. In some rare cases, this can cause tests to fail if they expect cloud definitions to come only from clouds.yaml. This change ignores these CI variables. Change-Id: I6458969b45f5534e1172b9f8ba598d7c536a59b5 --- os_client_config/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1840d18dc..7c65b537b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -82,7 +82,11 @@ def get_boolean(value): def _get_os_environ(): ret = defaults.get_defaults() - environkeys = [k for k in os.environ.keys() if k.startswith('OS_')] + environkeys = [k for k in os.environ.keys() + if k.startswith('OS_') + and not k.startswith('OS_TEST') # infra CI var + and not k.startswith('OS_STD') # infra CI var + ] if not environkeys: return None for k in environkeys: From 0e3117cb01d201f403be72cbb1218b9eec69a541 Mon Sep 17 00:00:00 2001 From: lifeless Date: Thu, 13 Aug 2015 07:09:22 +0000 Subject: [PATCH 119/365] Revert "Use the correct auth_plugin for token authentication" This reverts commit dfef01a7cbb9d68ad443d182e548ec58143a3a1f. This broke the world. Change-Id: I65d820b0da807a161eae814ea4b5413f245d7bf0 --- os_client_config/config.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 240220ae9..1840d18dc 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -379,12 +379,8 @@ class OpenStackConfig(object): def _validate_auth(self, config): # May throw a keystoneclient.exceptions.NoMatchingPlugin - if config['auth_type'] == 'token_endpoint': - auth_plugin = ksc_auth.token_endpoint.Token - else: - auth_plugin = ksc_auth.get_plugin_class(config['auth_type']) - - plugin_options = auth_plugin.get_options() + plugin_options = ksc_auth.get_plugin_class( + config['auth_type']).get_options() for p_opt in plugin_options: # if it's in config.auth, win, kill it from config dict From e29ed75a4b72ed25c7791dab8a66f3a3c86f962a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 13 Aug 2015 09:14:01 +0000 Subject: [PATCH 120/365] Revert "Revert "Use the correct auth_plugin for token authentication"" Name the auth type needed for using an admin token "admin_token". This hack can be removed with keystoneclient and/or keystoneauth make a release with the admin_token entrypoint defined. Naming it admin_token further reduces the conflict with OSC, which should make the patch to work around OSC having different arguments unnecessary. This reverts commit 0e3117cb01d201f403be72cbb1218b9eec69a541. Change-Id: I079c7c61b2637ded73542876cb1378a0731f8631 --- os_client_config/config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1840d18dc..99839785e 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -379,8 +379,12 @@ class OpenStackConfig(object): def _validate_auth(self, config): # May throw a keystoneclient.exceptions.NoMatchingPlugin - plugin_options = ksc_auth.get_plugin_class( - config['auth_type']).get_options() + if config['auth_type'] == 'admin_endpoint': + auth_plugin = ksc_auth.token_endpoint.Token + else: + auth_plugin = ksc_auth.get_plugin_class(config['auth_type']) + + plugin_options = auth_plugin.get_options() for p_opt in plugin_options: # if it's in config.auth, win, kill it from config dict From 2726843ee54a3c9f061b9ed2451d080f459e4f38 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 18 Aug 2015 16:27:08 -0700 Subject: [PATCH 121/365] Do not treat project_name and project_id the same There are clouds where doing this is not working. Change-Id: I1d2e71b2a6ad22eb5070b92448779f2e9df71e4a --- os_client_config/config.py | 6 +++--- os_client_config/tests/base.py | 6 +++++- os_client_config/tests/test_config.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1542f2f0a..d6983781f 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -288,9 +288,9 @@ class OpenStackConfig(object): def _fix_backwards_project(self, cloud): # Do the lists backwards so that project_name is the ultimate winner mappings = { - 'project_name': ('tenant_id', 'tenant-id', - 'project_id', 'project-id', - 'tenant_name', 'tenant-name', + 'project_id': ('tenant_id', 'tenant-id', + 'project_id', 'project-id'), + 'project_name': ('tenant_name', 'tenant-name', 'project_name', 'project-name'), } for target_key, possible_values in mappings.items(): diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 569b52ea7..89e04c0ae 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -132,4 +132,8 @@ class TestCase(base.BaseTestCase): self.assertIsNone(cc.cloud) self.assertIn('username', cc.auth) self.assertEqual('testuser', cc.auth['username']) - self.assertEqual('testproject', cc.auth['project_name']) + self.assertTrue('project_name' in cc.auth or 'project_id' in cc.auth) + if 'project_name' in cc.auth: + self.assertEqual('testproject', cc.auth['project_name']) + elif 'project_id' in cc.auth: + self.assertEqual('testproject', cc.auth['project_id']) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 6bb65fce5..332e4d323 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -89,13 +89,13 @@ class TestConfig(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud('_test-cloud-int-project_') - self.assertEqual('12345', cc.auth['project_name']) + self.assertEqual('12345', cc.auth['project_id']) def test_get_one_cloud_with_hyphenated_project_id(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud('_test_cloud_hyphenated') - self.assertEqual('12345', cc.auth['project_name']) + self.assertEqual('12345', cc.auth['project_id']) def test_no_environ(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], From fb24f1d113c03cf4d542331428e63f58a80eef71 Mon Sep 17 00:00:00 2001 From: Eric Harney Date: Thu, 3 Sep 2015 07:52:37 -0400 Subject: [PATCH 122/365] Handle empty defaults.yaml file If defaults.yaml is empty, a TypeError is thrown because the result of yaml.load is not iterable. Change-Id: Ic3283ebaf9dd325e4f430e70bce08c6d716f60bc --- os_client_config/defaults.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index a274767bd..897cff568 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -35,6 +35,8 @@ def get_defaults(): key=None, ) with open(_yaml_path, 'r') as yaml_file: - _defaults.update(yaml.load(yaml_file.read())) + updates = yaml.load(yaml_file.read()) + if updates is not None: + _defaults.update(updates) return _defaults.copy() From 24bff5dca48365a767d8b6f0c38be75b2d67d6fd Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 4 Sep 2015 09:41:57 -0400 Subject: [PATCH 123/365] Update OVH public cloud information OVH has two regions now. Also, be clear that OVH does not use floating ips. Change-Id: Iabd84748cccec7da6e083b647511cba48d57bc20 --- doc/source/vendor-support.rst | 1 + os_client_config/vendors/ovh.yaml | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 3cf39e03f..2af8f6eb3 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -153,6 +153,7 @@ https://auth.cloud.ovh.net/v2.0 Region Name Human Name ============== ================ SBG-1 Strassbourg, FR +GRA-1 Gravelines, FR ============== ================ * Images must be in `raw` format diff --git a/os_client_config/vendors/ovh.yaml b/os_client_config/vendors/ovh.yaml index f83437297..52b91a466 100644 --- a/os_client_config/vendors/ovh.yaml +++ b/os_client_config/vendors/ovh.yaml @@ -2,5 +2,8 @@ name: ovh profile: auth: auth_url: https://auth.cloud.ovh.net/v2.0 - region_name: SBG1 + regions: + - GRA1 + - SBG1 image_format: raw + floating_ip_source: None From a51ab85bff1172dda04ce57021ad87041b754bb5 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 12 Sep 2015 09:54:16 +0200 Subject: [PATCH 124/365] Add default version number for heat Heat only has v1. Change-Id: I31b40b7cfec0685105b8066d1cb385befae00f90 --- os_client_config/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/os_client_config/defaults.yaml b/os_client_config/defaults.yaml index 9130228f2..abc75cacc 100644 --- a/os_client_config/defaults.yaml +++ b/os_client_config/defaults.yaml @@ -12,5 +12,6 @@ image_api_version: '1' image_format: qcow2 network_api_version: '2' object_api_version: '1' +orchestration_api_version: '1' secgroup_source: neutron volume_api_version: '1' From eb6ed09dede71a441d005d773b94411d70f1557d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 02:11:14 +0200 Subject: [PATCH 125/365] Remove duplicate lines that are the same as default Change-Id: I007802ac08a8708fbbbfa8ee8c7d79e4e2bb55f7 --- doc/source/vendor-support.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 2af8f6eb3..e4db27665 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -109,7 +109,6 @@ BHS-1 Beauharnois, QC ============== ================ * Image API Version is 2 -* Images must be in `qcow2` format * Floating IPs are not supported unitedstack @@ -139,7 +138,6 @@ Region Name Human Name RegionOne RegionOne ============== ================ -* Identity API Version is 2 * Public IPv4 is provided via NAT with Nova Floating IP * Floating IPs are provided by Nova * Security groups are provided by Nova From 15c6652dc4bfc5b417bb67df0435935962d71d9c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 7 Sep 2015 18:55:24 -0500 Subject: [PATCH 126/365] Return keystoneauth plugins based on auth args We know all of the things that we need to know to return an appropriate auth plugin from keystoneauth based on the auth parameters. This also introduces a hard-depend on keystoneauth, which should be fine since keystoneauth itself has a very low dependency count. We'll also use ksa to help validate auth parameters when we're doing the config processing. Change-Id: Ia1a1a4adb4dcefed5d7607082e026ca7361f394d --- os_client_config/cloud_config.py | 16 ++++++++++++++++ requirements.txt | 1 + 2 files changed, 17 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index f60303b2a..c9f9f0756 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -14,6 +14,8 @@ import warnings +from keystoneauth1 import loading + class CloudConfig(object): def __init__(self, name, region, config, prefer_ipv6=False): @@ -106,3 +108,17 @@ class CloudConfig(object): @property def prefer_ipv6(self): return self._prefer_ipv6 + + def get_auth(self): + """Return a keystoneauth plugin from the auth credentials.""" + # Re-use the admin_token plugin for the "None" plugin + # since it does not look up endpoints or tokens but rather + # does a passthrough. This is useful for things like Ironic + # that have a keystoneless operational mode, but means we're + # still dealing with a keystoneauth Session object, so all the + # _other_ things (SSL arg handling, timeout) all work consistently + if self.config['auth_type'] in (None, "None", ''): + self.config['auth_type'] = 'admin_token' + self.config['auth']['token'] = None + loader = loading.get_plugin_loader(self.config['auth_type']) + return loader.load_from_options(**self.config['auth']) diff --git a/requirements.txt b/requirements.txt index 894a70acc..db0b6354a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ # process, which may cause wedges in the gate later. PyYAML>=3.1.0 appdirs>=1.3.0 +keystoneauth1>=1.0.0 From 53b7b7a6d61243254974ad3c1c256809c86a27bf Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 02:09:12 +0200 Subject: [PATCH 127/365] Add citycloud to the vendors list Nice job with keystone v3 support. Change-Id: Ib2b54feef5055b04740a63a3a1d4e0b967018864 --- doc/source/vendor-support.rst | 17 +++++++++++++++++ os_client_config/vendors/citycloud.yaml | 9 +++++++++ 2 files changed, 26 insertions(+) create mode 100644 os_client_config/vendors/citycloud.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index e4db27665..51e843a96 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -156,3 +156,20 @@ GRA-1 Gravelines, FR * Images must be in `raw` format * Floating IPs are not supported + +citycloud +--------- + +https://identity1.citycloud.com:5000/v3/ + +============== ================ +Region Name Human Name +============== ================ +Lon1 London, UK +Sto2 Stockholm, SE +Kna1 Karlskrona, SE +============== ================ + +* Identity API Version is 3 +* Image API Version is 2 +* Public IPv4 is provided via NAT with Neutron Floating IP diff --git a/os_client_config/vendors/citycloud.yaml b/os_client_config/vendors/citycloud.yaml new file mode 100644 index 000000000..9ee0b9b4f --- /dev/null +++ b/os_client_config/vendors/citycloud.yaml @@ -0,0 +1,9 @@ +name: citycloud +profile: + auth: + auth_url: https://identity1.citycloud.com:5000/v3/ + regions: + - Lon1 + - Sto2 + - Kna1 + identity_api_version: '3' From d11d165e697871b73236025efd5e8698d0b08750 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 02:30:13 +0200 Subject: [PATCH 128/365] Update auro auth_url and region information Also, turns out auro is running Glance v2. Change-Id: Iccc2e09f45192ac21001c346dab048c77a0f7813 --- doc/source/vendor-support.rst | 3 ++- os_client_config/vendors/auro.yaml | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 51e843a96..58207a7dc 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -135,10 +135,11 @@ https://api.auro.io:5000/v2.0 ============== ================ Region Name Human Name ============== ================ -RegionOne RegionOne +van1 Vancouver, BC ============== ================ * Public IPv4 is provided via NAT with Nova Floating IP +* Image API Version is 2 * Floating IPs are provided by Nova * Security groups are provided by Nova diff --git a/os_client_config/vendors/auro.yaml b/os_client_config/vendors/auro.yaml index 987838bbb..c19a6571a 100644 --- a/os_client_config/vendors/auro.yaml +++ b/os_client_config/vendors/auro.yaml @@ -1,7 +1,8 @@ name: auro profile: auth: - auth_url: https://api.auro.io:5000/v2.0 - region_name: RegionOne + auth_url: https://api.van1.auro.io:5000/v2.0 + region_name: van1 + image_api_version: '2' secgroup_source: nova floating_ip_source: nova From 265abb2bf6e050d9c7bf9b3e0a307dca32899c1f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 02:13:06 +0200 Subject: [PATCH 129/365] Switch the image default to v2 It turns out it's more common than v1. Also, ovh was inappropriately listing glance v1. Change-Id: Icb534bf98fd3fa1c900ed9e4dd09ea0643176ff9 --- doc/source/vendor-support.rst | 11 ++--------- os_client_config/defaults.yaml | 2 +- os_client_config/vendors/dreamhost.yaml | 1 - os_client_config/vendors/hp.yaml | 1 + os_client_config/vendors/rackspace.yaml | 1 - os_client_config/vendors/runabove.yaml | 1 - os_client_config/vendors/unitedstack.yaml | 1 - os_client_config/vendors/vexxhost.yaml | 1 - 8 files changed, 4 insertions(+), 15 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 58207a7dc..b8ececbfb 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -15,7 +15,7 @@ These are the default behaviors unless a cloud is configured differently. * Identity uses `password` authentication * Identity API Version is 2 -* Image API Version is 1 +* Image API Version is 2 * Images must be in `qcow2` format * Images are uploaded using PUT interface * Public IPv4 is directly routable via DHCP from Neutron @@ -37,6 +37,7 @@ region-b.geo-1 US East ============== ================ * DNS Service Type is `hpext:dns` +* Image API Version is 1 * Public IPv4 is provided via NAT with Neutron Floating IP rackspace @@ -56,7 +57,6 @@ HKG Hong Kong * Database Service Type is `rax:database` * Compute Service Name is `cloudServersOpenStack` -* Image API Version is 2 * Images must be in `vhd` format * Images must be uploaded using the Glance Task Interface * Floating IPs are not needed @@ -78,7 +78,6 @@ Region Name Human Name RegionOne Region One ============== ================ -* Image API Version is 2 * Images must be in `raw` format * Public IPv4 is provided via NAT with Neutron Floating IP * IPv6 is provided to every server @@ -94,8 +93,6 @@ Region Name Human Name ca-ymq-1 Montreal ============== ================ -* Image API Version is 2 - runabove -------- @@ -108,7 +105,6 @@ SBG-1 Strassbourg, FR BHS-1 Beauharnois, QC ============== ================ -* Image API Version is 2 * Floating IPs are not supported unitedstack @@ -124,7 +120,6 @@ gd1 Guangdong ============== ================ * Identity API Version is 3 -* Image API Version is 2 * Images must be in `raw` format auro @@ -139,7 +134,6 @@ van1 Vancouver, BC ============== ================ * Public IPv4 is provided via NAT with Nova Floating IP -* Image API Version is 2 * Floating IPs are provided by Nova * Security groups are provided by Nova @@ -172,5 +166,4 @@ Kna1 Karlskrona, SE ============== ================ * Identity API Version is 3 -* Image API Version is 2 * Public IPv4 is provided via NAT with Neutron Floating IP diff --git a/os_client_config/defaults.yaml b/os_client_config/defaults.yaml index abc75cacc..e81eb1649 100644 --- a/os_client_config/defaults.yaml +++ b/os_client_config/defaults.yaml @@ -8,7 +8,7 @@ interface: public floating_ip_source: neutron identity_api_version: '2' image_api_use_tasks: false -image_api_version: '1' +image_api_version: '2' image_format: qcow2 network_api_version: '2' object_api_version: '1' diff --git a/os_client_config/vendors/dreamhost.yaml b/os_client_config/vendors/dreamhost.yaml index 5e10d1403..3cd395a17 100644 --- a/os_client_config/vendors/dreamhost.yaml +++ b/os_client_config/vendors/dreamhost.yaml @@ -3,5 +3,4 @@ profile: auth: auth_url: https://keystone.dream.io/v2.0 region_name: RegionOne - image_api_version: '2' image_format: raw diff --git a/os_client_config/vendors/hp.yaml b/os_client_config/vendors/hp.yaml index 4da49568d..a0544df82 100644 --- a/os_client_config/vendors/hp.yaml +++ b/os_client_config/vendors/hp.yaml @@ -6,3 +6,4 @@ profile: - region-a.geo-1 - region-b.geo-1 dns_service_type: hpext:dns + image_api_version: '1' diff --git a/os_client_config/vendors/rackspace.yaml b/os_client_config/vendors/rackspace.yaml index ad7825dec..5cf7a44e6 100644 --- a/os_client_config/vendors/rackspace.yaml +++ b/os_client_config/vendors/rackspace.yaml @@ -10,7 +10,6 @@ profile: - SYD database_service_type: rax:database compute_service_name: cloudServersOpenStack - image_api_version: '2' image_api_use_tasks: true image_format: vhd floating_ip_source: None diff --git a/os_client_config/vendors/runabove.yaml b/os_client_config/vendors/runabove.yaml index 57e75f37a..34528941a 100644 --- a/os_client_config/vendors/runabove.yaml +++ b/os_client_config/vendors/runabove.yaml @@ -5,6 +5,5 @@ profile: regions: - BHS-1 - SBG-1 - image_api_version: '2' image_format: qcow2 floating_ip_source: None diff --git a/os_client_config/vendors/unitedstack.yaml b/os_client_config/vendors/unitedstack.yaml index f22bd8abe..c6d5cc20e 100644 --- a/os_client_config/vendors/unitedstack.yaml +++ b/os_client_config/vendors/unitedstack.yaml @@ -6,6 +6,5 @@ profile: - bj1 - gd1 identity_api_version: '3' - image_api_version: '2' image_format: raw floating_ip_source: None diff --git a/os_client_config/vendors/vexxhost.yaml b/os_client_config/vendors/vexxhost.yaml index f67c644c6..4a0ba271b 100644 --- a/os_client_config/vendors/vexxhost.yaml +++ b/os_client_config/vendors/vexxhost.yaml @@ -3,5 +3,4 @@ profile: auth: auth_url: http://auth.api.thenebulacloud.com:5000/v2.0/ region_name: ca-ymq-1 - image_api_version: '2' floating_ip_source: None From 7bfa633e86e9ccef6cd58472f6994b05f95ad032 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 11:54:22 +0200 Subject: [PATCH 130/365] Add elastx to vendor support matrix Change-Id: I67973f89e2da4ef550e46c32ad63a4c8e043b4d0 --- doc/source/vendor-support.rst | 13 +++++++++++++ os_client_config/vendors/elastx.yaml | 5 +++++ 2 files changed, 18 insertions(+) create mode 100644 os_client_config/vendors/elastx.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index b8ececbfb..f4179e783 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -167,3 +167,16 @@ Kna1 Karlskrona, SE * Identity API Version is 3 * Public IPv4 is provided via NAT with Neutron Floating IP + +elastx +------ + +https://ops.elastx.net:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +regionOne Region One +============== ================ + +* Public IPv4 is provided via NAT with Neutron Floating IP diff --git a/os_client_config/vendors/elastx.yaml b/os_client_config/vendors/elastx.yaml new file mode 100644 index 000000000..810e12ede --- /dev/null +++ b/os_client_config/vendors/elastx.yaml @@ -0,0 +1,5 @@ +name: elastx +profile: + auth: + auth_url: https://ops.elastx.net:5000/v2.0 + region_name: regionOne From d835c47ad35a27bb0e90a8db42d5c943a65c6608 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 12:54:49 +0200 Subject: [PATCH 131/365] Add Enter Cloud Suite to vendors list Change-Id: I78e9191073e68a9d3f78ba11b47aa0b1ff816430 --- doc/source/vendor-support.rst | 13 +++++++++++++ os_client_config/vendors/entercloudsuite.yaml | 8 ++++++++ 2 files changed, 21 insertions(+) create mode 100644 os_client_config/vendors/entercloudsuite.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index f4179e783..d987b1cd2 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -180,3 +180,16 @@ regionOne Region One ============== ================ * Public IPv4 is provided via NAT with Neutron Floating IP + +entercloudsuite +--------------- + +https://api.entercloudsuite.com/v2.0 + +============== ================ +Region Name Human Name +============== ================ +nl-ams1 Amsterdam, NL +it-mil1 Milan, IT +de-fra1 Frankfurt, DE +============== ================ diff --git a/os_client_config/vendors/entercloudsuite.yaml b/os_client_config/vendors/entercloudsuite.yaml new file mode 100644 index 000000000..f68bcf674 --- /dev/null +++ b/os_client_config/vendors/entercloudsuite.yaml @@ -0,0 +1,8 @@ +name: entercloudsuite +profile: + auth: + auth_url: https://api.entercloudsuite.com/v2.0 + regions: + - it-mil1 + - nl-ams1 + - de-fra1 From 2762241d7bfccf8c4bad471dfc20528d1135602d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 13:14:32 +0200 Subject: [PATCH 132/365] Add ultimum to list of vendors Change-Id: Ic5b5ee307b9d1eb338839a7feeeab50469316cf5 --- doc/source/vendor-support.rst | 11 +++++++++++ os_client_config/vendors/ultimum.yaml | 5 +++++ 2 files changed, 16 insertions(+) create mode 100644 os_client_config/vendors/ultimum.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index d987b1cd2..c0fc8c596 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -193,3 +193,14 @@ nl-ams1 Amsterdam, NL it-mil1 Milan, IT de-fra1 Frankfurt, DE ============== ================ + +ultimum +------- + +https://console.ultimum-cloud.com:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +RegionOne Region One +============== ================ diff --git a/os_client_config/vendors/ultimum.yaml b/os_client_config/vendors/ultimum.yaml new file mode 100644 index 000000000..866117491 --- /dev/null +++ b/os_client_config/vendors/ultimum.yaml @@ -0,0 +1,5 @@ +name: ultimum +profile: + auth: + auth_url: https://console.ultimum-cloud.com:5000/v2.0 + region-name: RegionOne From b0f4161423d683aad2e5006ba2f9d80e858928ed Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 14:00:25 +0200 Subject: [PATCH 133/365] Add Datacentred to the vendor list Change-Id: Iac2a51f2351e6b46469508abadd63942829663f0 --- doc/source/vendor-support.rst | 15 ++++++++++++++- os_client_config/vendors/datacentred.yaml | 6 ++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 os_client_config/vendors/datacentred.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index c0fc8c596..4d253547f 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -202,5 +202,18 @@ https://console.ultimum-cloud.com:5000/v2.0 ============== ================ Region Name Human Name ============== ================ -RegionOne Region One +RegionOne Region One ============== ================ + +datacentred +----------- + +https://compute.datacentred.io:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +sal01 Manchester, UK +============== ================ + +* Image API Version is 1 diff --git a/os_client_config/vendors/datacentred.yaml b/os_client_config/vendors/datacentred.yaml new file mode 100644 index 000000000..5c0a5ed1a --- /dev/null +++ b/os_client_config/vendors/datacentred.yaml @@ -0,0 +1,6 @@ +name: datacentred +profile: + auth: + auth_url: https://compute.datacentred.io:5000/v2.0 + region-name: sal01 + image_api_version: '1' From 3f76cc5fa9d288b25b3804966568b8bd900a54c1 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 15 Sep 2015 14:02:01 +0200 Subject: [PATCH 134/365] Remove an extra line Change-Id: I3a7f31a205b9a49e8c93f71e2a9dacacee6e2f91 --- os_client_config/vendors/auro.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/os_client_config/vendors/auro.yaml b/os_client_config/vendors/auro.yaml index c19a6571a..ad721e62d 100644 --- a/os_client_config/vendors/auro.yaml +++ b/os_client_config/vendors/auro.yaml @@ -3,6 +3,5 @@ profile: auth: auth_url: https://api.van1.auro.io:5000/v2.0 region_name: van1 - image_api_version: '2' secgroup_source: nova floating_ip_source: nova From 093d7b085a5743d21f74da532eb45a916830022c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 7 Sep 2015 20:26:24 -0500 Subject: [PATCH 135/365] Defer plugin validation to keystoneauth keystoneauth plugin loading has parameter validation itself. Rather than us doing it, let ksa do it. This bubbles up a ksa exception- but I think I'm ok with that as an interface. Change-Id: I3e7741a1b623b133f24f321e97539883dc6cd153 --- os_client_config/cloud_config.py | 18 ++------ os_client_config/config.py | 65 +++++++++++++-------------- os_client_config/tests/base.py | 6 +++ os_client_config/tests/test_config.py | 24 +++++----- 4 files changed, 55 insertions(+), 58 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index c9f9f0756..86b4f5073 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -14,15 +14,15 @@ import warnings -from keystoneauth1 import loading - class CloudConfig(object): - def __init__(self, name, region, config, prefer_ipv6=False): + def __init__(self, name, region, config, + prefer_ipv6=False, auth_plugin=None): self.name = name self.region = region self.config = config self._prefer_ipv6 = prefer_ipv6 + self._auth = auth_plugin def __getattr__(self, key): """Return arbitrary attributes.""" @@ -111,14 +111,4 @@ class CloudConfig(object): def get_auth(self): """Return a keystoneauth plugin from the auth credentials.""" - # Re-use the admin_token plugin for the "None" plugin - # since it does not look up endpoints or tokens but rather - # does a passthrough. This is useful for things like Ironic - # that have a keystoneless operational mode, but means we're - # still dealing with a keystoneauth Session object, so all the - # _other_ things (SSL arg handling, timeout) all work consistently - if self.config['auth_type'] in (None, "None", ''): - self.config['auth_type'] = 'admin_token' - self.config['auth']['token'] = None - loader = loading.get_plugin_loader(self.config['auth_type']) - return loader.load_from_options(**self.config['auth']) + return self._auth diff --git a/os_client_config/config.py b/os_client_config/config.py index d6983781f..5da7799dd 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -17,13 +17,9 @@ import os import warnings import appdirs +from keystoneauth1 import loading import yaml -try: - import keystoneclient.auth as ksc_auth -except ImportError: - ksc_auth = None - from os_client_config import cloud_config from os_client_config import defaults from os_client_config import exceptions @@ -376,19 +372,27 @@ class OpenStackConfig(object): if opt_name in config: return config[opt_name] else: - for d_opt in opt.deprecated_opts: + for d_opt in opt.deprecated: d_opt_name = d_opt.name.replace('-', '_') if d_opt_name in config: return config[d_opt_name] - def _validate_auth(self, config): - # May throw a keystoneclient.exceptions.NoMatchingPlugin - if config['auth_type'] == 'admin_endpoint': - auth_plugin = ksc_auth.token_endpoint.Token - else: - auth_plugin = ksc_auth.get_plugin_class(config['auth_type']) + def _get_auth_loader(self, config): + # Re-use the admin_token plugin for the "None" plugin + # since it does not look up endpoints or tokens but rather + # does a passthrough. This is useful for things like Ironic + # that have a keystoneless operational mode, but means we're + # still dealing with a keystoneauth Session object, so all the + # _other_ things (SSL arg handling, timeout) all work consistently + if config['auth_type'] in (None, "None", ''): + config['auth_type'] = 'admin_token' + config['auth']['token'] = None + return loading.get_plugin_loader(config['auth_type']) - plugin_options = auth_plugin.get_options() + def _validate_auth(self, config, loader): + # May throw a keystoneclient.exceptions.NoMatchingPlugin + + plugin_options = loader.get_options() for p_opt in plugin_options: # if it's in config.auth, win, kill it from config dict @@ -400,19 +404,8 @@ class OpenStackConfig(object): if not winning_value: winning_value = self._find_winning_auth_value(p_opt, config) - # if the plugin tells us that this value is required - # then error if it's doesn't exist now - if not winning_value and p_opt.required: - raise exceptions.OpenStackConfigException( - 'Unable to find auth information for cloud' - ' {cloud} in config files {files}' - ' or environment variables. Missing value {auth_key}' - ' required for auth plugin {plugin}'.format( - cloud=cloud, files=','.join(self._config_files), - auth_key=p_opt.name, plugin=config.get('auth_type'))) - # Clean up after ourselves - for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: + for opt in [p_opt.name] + [o.name for o in p_opt.deprecated]: opt = opt.replace('-', '_') config.pop(opt, None) config['auth'].pop(opt, None) @@ -435,13 +428,16 @@ class OpenStackConfig(object): :param string cloud: The name of the configuration to load from clouds.yaml :param boolean validate: - Validate that required arguments are present and certain - argument combinations are valid + Validate the config. Setting this to False causes no auth plugin + to be created. It's really only useful for testing. :param Namespace argparse: An argparse Namespace object; allows direct passing in of argparse options to be added to the cloud config. Values of None and '' will be removed. :param kwargs: Additional configuration options + + :raises: keystoneauth1.exceptions.MissingRequiredOptions + on missing required auth parameters """ if cloud is None and self.envvar_key in self.get_cloud_names(): @@ -471,12 +467,12 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) - if 'auth_type' in config: - if config['auth_type'] in ('', 'None', None): - validate = False - - if validate and ksc_auth: - config = self._validate_auth(config) + loader = self._get_auth_loader(config) + if validate: + config = self._validate_auth(config, loader) + auth_plugin = loader.load_from_options(**config['auth']) + else: + auth_plugin = None # If any of the defaults reference other values, we need to expand for (key, value) in config.items(): @@ -492,7 +488,8 @@ class OpenStackConfig(object): return cloud_config.CloudConfig( name=cloud_name, region=config['region_name'], config=self._normalize_keys(config), - prefer_ipv6=prefer_ipv6) + prefer_ipv6=prefer_ipv6, + auth_plugin=auth_plugin) @staticmethod def set_one_cloud(config_file, cloud, set_config=None): diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 89e04c0ae..9a2923792 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -31,6 +31,7 @@ VENDOR_CONF = { 'public-clouds': { '_test_cloud_in_our_cloud': { 'auth': { + 'auth_url': 'http://example.com/v2', 'username': 'testotheruser', 'project_name': 'testproject', }, @@ -45,6 +46,7 @@ USER_CONF = { '_test-cloud_': { 'profile': '_test_cloud_in_our_cloud', 'auth': { + 'auth_url': 'http://example.com/v2', 'username': 'testuser', 'password': 'testpass', }, @@ -53,6 +55,7 @@ USER_CONF = { '_test_cloud_no_vendor': { 'profile': '_test_non_existant_cloud', 'auth': { + 'auth_url': 'http://example.com/v2', 'username': 'testuser', 'password': 'testpass', 'project_name': 'testproject', @@ -64,6 +67,7 @@ USER_CONF = { 'username': 'testuser', 'password': 'testpass', 'project_id': 12345, + 'auth_url': 'http://example.com/v2', }, 'region_name': 'test-region', }, @@ -72,6 +76,7 @@ USER_CONF = { 'username': 'testuser', 'password': 'testpass', 'project-id': 'testproject', + 'auth_url': 'http://example.com/v2', }, 'regions': [ 'region1', @@ -83,6 +88,7 @@ USER_CONF = { 'username': 'testuser', 'password': 'testpass', 'project-id': '12345', + 'auth_url': 'http://example.com/v2', }, 'region_name': 'test-region', } diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 332e4d323..82f2fb9d8 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -43,7 +43,7 @@ class TestConfig(base.TestCase): def test_get_one_cloud(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cloud = c.get_one_cloud() + cloud = c.get_one_cloud(validate=False) self.assertIsInstance(cloud, cloud_config.CloudConfig) self.assertEqual(cloud.name, '') @@ -61,12 +61,12 @@ class TestConfig(base.TestCase): ) def test_get_one_cloud_auth_override_defaults(self): - default_options = {'auth_type': 'token'} + default_options = {'compute_api_version': '4'} c = config.OpenStackConfig(config_files=[self.cloud_yaml], override_defaults=default_options) cc = c.get_one_cloud(cloud='_test-cloud_', auth={'username': 'user'}) self.assertEqual('user', cc.auth['username']) - self.assertEqual('token', cc.auth_type) + self.assertEqual('4', cc.compute_api_version) self.assertEqual( defaults._defaults['identity_api_version'], cc.identity_api_version, @@ -109,7 +109,7 @@ class TestConfig(base.TestCase): for k in os.environ.keys(): if k.startswith('OS_'): self.useFixture(fixtures.EnvironmentVariable(k)) - c.get_one_cloud(cloud='defaults') + c.get_one_cloud(cloud='defaults', validate=False) def test_prefer_ipv6_true(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], @@ -120,7 +120,7 @@ class TestConfig(base.TestCase): def test_prefer_ipv6_false(self): c = config.OpenStackConfig(config_files=[self.no_yaml], vendor_files=[self.no_yaml]) - cc = c.get_one_cloud(cloud='defaults') + cc = c.get_one_cloud(cloud='defaults', validate=False) self.assertFalse(cc.prefer_ipv6) def test_get_one_cloud_auth_merge(self): @@ -144,7 +144,7 @@ class TestConfig(base.TestCase): for k in os.environ.keys(): if k.startswith('OS_'): self.useFixture(fixtures.EnvironmentVariable(k)) - c.get_one_cloud(cloud='defaults') + c.get_one_cloud(cloud='defaults', validate=False) self.assertEqual(['defaults'], sorted(c.get_cloud_names())) def test_set_one_cloud_creates_file(self): @@ -168,7 +168,8 @@ class TestConfig(base.TestCase): resulting_cloud_config = { 'auth': { 'password': 'newpass', - 'username': 'testuser' + 'username': 'testuser', + 'auth_url': 'http://example.com/v2', }, 'cloud': 'new_cloud', 'profile': '_test_cloud_in_our_cloud', @@ -189,6 +190,10 @@ class TestConfigArgparse(base.TestCase): super(TestConfigArgparse, self).setUp() self.options = argparse.Namespace( + auth_url='http://example.com/v2', + username='user', + password='password', + project_name='project', region_name='other-test-region', snack_type='cookie', ) @@ -208,7 +213,6 @@ class TestConfigArgparse(base.TestCase): cc = c.get_one_cloud(cloud='', argparse=self.options) self.assertIsNone(cc.cloud) - self.assertNotIn('username', cc.auth) self.assertEqual(cc.region_name, 'other-test-region') self.assertEqual(cc.snack_type, 'cookie') @@ -270,11 +274,11 @@ class TestConfigDefault(base.TestCase): self.assertEqual('password', cc.auth_type) def test_set_default_before_init(self): - config.set_default('auth_type', 'token') + config.set_default('identity_api_version', '4') c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud(cloud='_test-cloud_', argparse=None) - self.assertEqual('token', cc.auth_type) + self.assertEqual('4', cc.identity_api_version) class TestBackwardsCompatibility(base.TestCase): From 2be0553eb0d168ea0dc57e098c8dcc86566b6622 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 16 Sep 2015 21:54:18 +0200 Subject: [PATCH 136/365] Fix typo in comment - we use ksa not ksc Change-Id: I9f35c26fc633b07442141443574ea9b7582036be --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 5da7799dd..ec35548f6 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -390,7 +390,7 @@ class OpenStackConfig(object): return loading.get_plugin_loader(config['auth_type']) def _validate_auth(self, config, loader): - # May throw a keystoneclient.exceptions.NoMatchingPlugin + # May throw a keystoneauth1.exceptions.NoMatchingPlugin plugin_options = loader.get_options() From 54afb28565cb1b24973a403f365081487a2484ec Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 16 Sep 2015 21:18:19 +0200 Subject: [PATCH 137/365] Add internap to the vendor list Change-Id: I4c98b72f039fd97a2c55ecbfa546fb908d1fe539 --- doc/source/vendor-support.rst | 16 ++++++++++++++++ os_client_config/vendors/internap.yaml | 10 ++++++++++ 2 files changed, 26 insertions(+) create mode 100644 os_client_config/vendors/internap.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 4d253547f..c4f5a4fe4 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -217,3 +217,19 @@ sal01 Manchester, UK ============== ================ * Image API Version is 1 + +internap +-------- + +https://identity.api.cloud.iweb.com/v2.0 + +============== ================ +Region Name Human Name +============== ================ +ams01 Amsterdam, NL +da01 Dallas, TX +nyj01 New York, NY +============== ================ + +* Image API Version is 1 +* Floating IPs are not supported diff --git a/os_client_config/vendors/internap.yaml b/os_client_config/vendors/internap.yaml new file mode 100644 index 000000000..48cd960eb --- /dev/null +++ b/os_client_config/vendors/internap.yaml @@ -0,0 +1,10 @@ +name: internap +profile: + auth: + auth_url: https://identity.api.cloud.iweb.com/v2.0 + regions: + - ams01 + - da01 + - nyj01 + image_api_version: '1' + floating_ip_source: None From aef90e7ec82326e63136be3ace1fa0a0590ee325 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 16 Sep 2015 12:30:12 +0200 Subject: [PATCH 138/365] Allow configuring domain id once In the most common case, a use has one and only one domain id that they care about, and it's associated with their user and project. Instead of making them set it for both user_domain_id and project_domain_id, allow for domain_{id, name} and then fill in any missing values for {user,project}_domain_{id,name} with the given value. This is mainly because I wind up with config files looking like this: user_domain_id: d0919bd5e8d74e49adf0e145807ffc38 project_domain_id: d0919bd5e8d74e49adf0e145807ffc38 Which offends my tender sensibilities. Change-Id: I12342dfa9f1b539a3fea5dd8874c42d027c59739 --- os_client_config/config.py | 23 +++++++++++++++++++++++ os_client_config/tests/base.py | 11 +++++++++++ os_client_config/tests/test_config.py | 12 +++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index ec35548f6..e8e76a54c 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -279,15 +279,38 @@ class OpenStackConfig(object): cloud = self._fix_backwards_project(cloud) cloud = self._fix_backwards_auth_plugin(cloud) cloud = self._fix_backwards_interface(cloud) + cloud = self._handle_domain_id(cloud) + return cloud + + def _handle_domain_id(self, cloud): + # Allow people to just specify domain once if it's the same + mappings = { + 'domain_id': ('user_domain_id', 'project_domain_id'), + 'domain_name': ('user_domain_name', 'project_domain_name'), + } + for target_key, possible_values in mappings.items(): + for key in possible_values: + if target_key in cloud['auth'] and key not in cloud['auth']: + cloud['auth'][key] = cloud['auth'][target_key] + cloud['auth'].pop(target_key, None) return cloud def _fix_backwards_project(self, cloud): # Do the lists backwards so that project_name is the ultimate winner + # Also handle moving domain names into auth so that domain mapping + # is easier mappings = { 'project_id': ('tenant_id', 'tenant-id', 'project_id', 'project-id'), 'project_name': ('tenant_name', 'tenant-name', 'project_name', 'project-name'), + 'domain_id': ('domain_id', 'domain-id'), + 'domain_name': ('domain_name', 'domain-name'), + 'user_domain_id': ('user_domain_id', 'user-domain-id'), + 'user_domain_name': ('user_domain_name', 'user-domain-name'), + 'project_domain_id': ('project_domain_id', 'project-domain-id'), + 'project_domain_name': ( + 'project_domain_name', 'project-domain-name'), } for target_key, possible_values in mappings.items(): target = None diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 9a2923792..cbf58da36 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -71,6 +71,17 @@ USER_CONF = { }, 'region_name': 'test-region', }, + '_test-cloud-domain-id_': { + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project_id': 12345, + 'auth_url': 'http://example.com/v2', + 'domain_id': '6789', + 'project_domain_id': '123456789', + }, + 'region_name': 'test-region', + }, '_test_cloud_regions': { 'auth': { 'username': 'testuser', diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 82f2fb9d8..36fe15de1 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -91,6 +91,15 @@ class TestConfig(base.TestCase): cc = c.get_one_cloud('_test-cloud-int-project_') self.assertEqual('12345', cc.auth['project_id']) + def test_get_one_cloud_with_domain_id(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test-cloud-domain-id_') + self.assertEqual('6789', cc.auth['user_domain_id']) + self.assertEqual('123456789', cc.auth['project_domain_id']) + self.assertNotIn('domain_id', cc.auth) + self.assertNotIn('domain-id', cc.auth) + def test_get_one_cloud_with_hyphenated_project_id(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) @@ -132,7 +141,8 @@ class TestConfig(base.TestCase): def test_get_cloud_names(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml]) self.assertEqual( - ['_test-cloud-int-project_', + ['_test-cloud-domain-id_', + '_test-cloud-int-project_', '_test-cloud_', '_test_cloud_hyphenated', '_test_cloud_no_vendor', From aabf1431a3e23a734431f56df4a8d6ac446509b3 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 17 Sep 2015 01:22:24 +0200 Subject: [PATCH 139/365] Test kwargs passing not just argparse Change-Id: Ic14365b1daec9a3f51d6a59db38604edb60865d4 --- os_client_config/tests/test_config.py | 31 +++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 36fe15de1..b4320ad6b 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -199,7 +199,7 @@ class TestConfigArgparse(base.TestCase): def setUp(self): super(TestConfigArgparse, self).setUp() - self.options = argparse.Namespace( + self.args = dict( auth_url='http://example.com/v2', username='user', password='password', @@ -207,6 +207,7 @@ class TestConfigArgparse(base.TestCase): region_name='other-test-region', snack_type='cookie', ) + self.options = argparse.Namespace(**self.args) def test_get_one_cloud_argparse(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], @@ -221,7 +222,33 @@ class TestConfigArgparse(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud(cloud='', argparse=self.options) + cc = c.get_one_cloud(argparse=self.options) + self.assertIsNone(cc.cloud) + self.assertEqual(cc.region_name, 'other-test-region') + self.assertEqual(cc.snack_type, 'cookie') + + def test_get_one_cloud_just_kwargs(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud(**self.args) + self.assertIsNone(cc.cloud) + self.assertEqual(cc.region_name, 'other-test-region') + self.assertEqual(cc.snack_type, 'cookie') + + def test_get_one_cloud_dash_kwargs(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + args = { + 'auth-url': 'http://example.com/v2', + 'username': 'user', + 'password': 'password', + 'project_name': 'project', + 'region_name': 'other-test-region', + 'snack_type': 'cookie', + } + cc = c.get_one_cloud(**args) self.assertIsNone(cc.cloud) self.assertEqual(cc.region_name, 'other-test-region') self.assertEqual(cc.snack_type, 'cookie') From 2906d1e61fd6e2e1024996993e1c108966fc425f Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 16 Sep 2015 16:41:58 -0700 Subject: [PATCH 140/365] Properly handle os- prefixed args in fix_args Prior to this any arg that started with 'os' had the 'os' prefix chomped including the next character. This meant that any arg like 'osmosis' would've been replaced with 'osis' which is clearly wrong. Instead we want to treat the OpenStack prefix of 'os-' or 'os_' as the special thing so check the next character is correct before chomping. Change-Id: Id12a92adf63d896f7aa5c0e391abd299c4ce3331 --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index e8e76a54c..65312054f 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -383,7 +383,7 @@ class OpenStackConfig(object): new_args = dict() for (key, val) in iter(args.items()): key = key.replace('-', '_') - if key.startswith('os'): + if key.startswith('os_'): os_args[key[3:]] = val else: new_args[key] = val From 0339a9a7f8c004b7e6df5c3d9dc49a69fcc3b357 Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Wed, 16 Sep 2015 17:22:39 -0700 Subject: [PATCH 141/365] Convert auth kwargs '-' to '_' When passing in kwargs to _get_one_cloud we do not dive into the auth dict to convert '-' to '_'. Change-Id: I8ce12370b5fd4444ba17d724e7f8036a7b0d2784 --- os_client_config/config.py | 5 +++++ os_client_config/tests/test_config.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index e8e76a54c..100cf739e 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -382,6 +382,11 @@ class OpenStackConfig(object): os_args = dict() new_args = dict() for (key, val) in iter(args.items()): + if type(args[key]) == dict: + # dive into the auth dict + new_args[key] = self._fix_args(args[key]) + continue + key = key.replace('-', '_') if key.startswith('os'): os_args[key[3:]] = val diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 36fe15de1..9c9451d36 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -106,6 +106,21 @@ class TestConfig(base.TestCase): cc = c.get_one_cloud('_test_cloud_hyphenated') self.assertEqual('12345', cc.auth['project_id']) + def test_get_one_cloud_with_hyphenated_kwargs(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + args = { + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project-id': '12345', + 'auth-url': 'http://example.com/v2', + }, + 'region_name': 'test-region', + } + cc = c.get_one_cloud(**args) + self.assertEqual('http://example.com/v2', cc.auth['auth_url']) + def test_no_environ(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From 21bb2f347d0d52defa98726b7102c1853b558072 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 18 Sep 2015 09:55:55 +0200 Subject: [PATCH 142/365] Move plugin loader creation to try block We only need the plugin loader if we're going to create an auth plugin. Doing this when validate=False is clearly not a workable solution, because we'll wind up validating unknown plugins. Change-Id: Ieed44aa3ef41a14edd7529ca599a01967d517207 --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 387d01496..5dbe8db77 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -495,8 +495,8 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) - loader = self._get_auth_loader(config) if validate: + loader = self._get_auth_loader(config) config = self._validate_auth(config, loader) auth_plugin = loader.load_from_options(**config['auth']) else: From 6d25a939bfdfeea14e1038c240092ebaedf33145 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Fri, 18 Sep 2015 13:29:12 -0700 Subject: [PATCH 143/365] Fix typo in ovh region names Change-Id: If8dee92917d1f5c1e349f02eba7661c39dbe7d0b --- doc/source/vendor-support.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index c4f5a4fe4..77d10ebba 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -145,8 +145,8 @@ https://auth.cloud.ovh.net/v2.0 ============== ================ Region Name Human Name ============== ================ -SBG-1 Strassbourg, FR -GRA-1 Gravelines, FR +SBG1 Strassbourg, FR +GRA1 Gravelines, FR ============== ================ * Images must be in `raw` format From aa41f9bfb10b6f75e1a3f7c599c8c06e7a03d389 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 18 Sep 2015 16:54:46 -0400 Subject: [PATCH 144/365] Fall back to keystoneclient arg processing For things not on keystoneauth yet, we need to move things into the auth dict using keystoneclient. Change-Id: Ia4500cc270b775f189048ccf667d5bfdc5dfcd14 --- os_client_config/config.py | 80 +++++++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 6 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 5dbe8db77..96d5fa74c 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -17,7 +17,10 @@ import os import warnings import appdirs -from keystoneauth1 import loading +try: + from keystoneauth1 import loading +except ImportError: + loading = None import yaml from os_client_config import cloud_config @@ -400,7 +403,9 @@ class OpenStackConfig(object): if opt_name in config: return config[opt_name] else: - for d_opt in opt.deprecated: + deprecated = getattr(opt, 'deprecated', getattr( + opt, 'deprecated_opts')) + for d_opt in deprecated: d_opt_name = d_opt.name.replace('-', '_') if d_opt_name in config: return config[d_opt_name] @@ -417,6 +422,54 @@ class OpenStackConfig(object): config['auth']['token'] = None return loading.get_plugin_loader(config['auth_type']) + def _validate_auth_ksc(self, config): + try: + import keystoneclient.auth as ksc_auth + except ImportError: + return config + + # May throw a keystoneclient.exceptions.NoMatchingPlugin + plugin_options = ksc_auth.get_plugin_class( + config['auth_type']).get_options() + + for p_opt in plugin_options: + # if it's in config.auth, win, kill it from config dict + # if it's in config and not in config.auth, move it + # deprecated loses to current + # provided beats default, deprecated or not + winning_value = self._find_winning_auth_value( + p_opt, config['auth']) + if not winning_value: + winning_value = self._find_winning_auth_value(p_opt, config) + + # if the plugin tells us that this value is required + # then error if it's doesn't exist now + if not winning_value and p_opt.required: + raise exceptions.OpenStackConfigException( + 'Unable to find auth information for cloud' + ' {cloud} in config files {files}' + ' or environment variables. Missing value {auth_key}' + ' required for auth plugin {plugin}'.format( + cloud=cloud, files=','.join(self._config_files), + auth_key=p_opt.name, plugin=config.get('auth_type'))) + + # Clean up after ourselves + for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: + opt = opt.replace('-', '_') + config.pop(opt, None) + config['auth'].pop(opt, None) + + if winning_value: + # Prefer the plugin configuration dest value if the value's key + # is marked as depreciated. + if p_opt.dest is None: + config['auth'][p_opt.name.replace('-', '_')] = ( + winning_value) + else: + config['auth'][p_opt.dest] = winning_value + + return config + def _validate_auth(self, config, loader): # May throw a keystoneauth1.exceptions.NoMatchingPlugin @@ -495,12 +548,27 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) - if validate: - loader = self._get_auth_loader(config) - config = self._validate_auth(config, loader) - auth_plugin = loader.load_from_options(**config['auth']) + if loading: + if validate: + try: + loader = self._get_auth_loader(config) + config = self._validate_auth(config, loader) + auth_plugin = loader.load_from_options(**config['auth']) + except Exception as e: + # We WANT the ksa exception normally + # but OSC can't handle it right now, so we try deferring + # to ksc. If that ALSO fails, it means there is likely + # a deeper issue, so we assume the ksa error was correct + auth_plugin = None + try: + config = self._validate_auth_ksc(config) + except Exception: + raise e + else: + auth_plugin = None else: auth_plugin = None + config = self._validate_auth_ksc(config) # If any of the defaults reference other values, we need to expand for (key, value) in config.items(): From 8dee656df809ed1b39b2800e35cc5ef67c31e84e Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 20 Sep 2015 20:08:39 -0400 Subject: [PATCH 145/365] Handle ksa opt with no deprecated field Bad logic fallthrough causes us to die when trying to process an option that does not have a deprecated option. Change-Id: I613466c6146a94b66a0a6d9955cdc4a6556f44ed --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 96d5fa74c..1b6193e46 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -404,7 +404,7 @@ class OpenStackConfig(object): return config[opt_name] else: deprecated = getattr(opt, 'deprecated', getattr( - opt, 'deprecated_opts')) + opt, 'deprecated_opts', [])) for d_opt in deprecated: d_opt_name = d_opt.name.replace('-', '_') if d_opt_name in config: From 7f33416020801dfc8a444a80705ef2f491a4d066 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 21 Sep 2015 14:41:32 +0000 Subject: [PATCH 146/365] Change ignore-errors to ignore_errors Needed for coverage 4.0 Change-Id: I033aaed4afa9037017190bc0b5aba7216840627d --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 5f0c7fd8b..3c1292222 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,4 +4,4 @@ source = os_client_config omit = os_client_config/tests/*,os_client_config/openstack/* [report] -ignore-errors = True \ No newline at end of file +ignore_errors = True From 75954fac2f711bfed815fd4015dff48ea26492c0 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Tue, 22 Sep 2015 09:23:12 +1200 Subject: [PATCH 147/365] Add support for Catalyst as vendor This adds Catalyst IT's public cloud to the list of vendors, https://catalyst.net.nz/catalyst-cloud Change-Id: I2d886bfc4fc94c534a55f6bad1120e474654524f --- doc/source/vendor-support.rst | 14 ++++++++++++++ os_client_config/vendors/catalyst.yaml | 10 ++++++++++ 2 files changed, 24 insertions(+) create mode 100644 os_client_config/vendors/catalyst.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 77d10ebba..39f488d14 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -233,3 +233,17 @@ nyj01 New York, NY * Image API Version is 1 * Floating IPs are not supported + +catalyst +-------- + +https://api.cloud.catalyst.net.nz:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +nz-por-1 Porirua, NZ +nz_wlg_1 Wellington, NZ + +* Image API Version is 1 +* Images must be in `raw` format diff --git a/os_client_config/vendors/catalyst.yaml b/os_client_config/vendors/catalyst.yaml new file mode 100644 index 000000000..348ceafc8 --- /dev/null +++ b/os_client_config/vendors/catalyst.yaml @@ -0,0 +1,10 @@ +name: catalyst +profile: + auth: + auth_url: https://api.cloud.catalyst.net.nz:5000/v2.0 + regions: + - nz-por-1 + - nz_wlg_1 + identity_api_version: '2' + image_api_version: '1' + image_format: raw From 0f089647b89817c5347e1f01335fd6aa6f9d313b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 22 Sep 2015 11:29:46 -0500 Subject: [PATCH 148/365] Fix a little error with the None auth type Ironic has a mode where it does not use auth, but we still funnel that through the ksa admin_token type for consistency of code. The hack we had to do that went too far and caused validate_auth to strip the fake token we were adding. Change-Id: Id5275ac7db1a6052db02c2286cbf88862cb1ff70 --- os_client_config/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1b6193e46..fcca30af3 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -419,7 +419,9 @@ class OpenStackConfig(object): # _other_ things (SSL arg handling, timeout) all work consistently if config['auth_type'] in (None, "None", ''): config['auth_type'] = 'admin_token' - config['auth']['token'] = None + # Set to notused rather than None because validate_auth will + # strip the value if it's actually python None + config['auth']['token'] = 'notused' return loading.get_plugin_loader(config['auth_type']) def _validate_auth_ksc(self, config): From 988e305b37e0f840ba09fdde882158641f8a1d05 Mon Sep 17 00:00:00 2001 From: Steve Martinelli Date: Tue, 22 Sep 2015 17:19:13 -0400 Subject: [PATCH 149/365] update RST for readme so pypi looks pretty Navigating to https://pypi.python.org/pypi/os-client-config results in seeing the raw RST content of the readme file. This is likely caused by minor RST warnings, but pypi gives up and shows it raw. Change-Id: Ia2d6202ade5282d9aeae9bb948175aae2aa264cd --- README.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 4cae7358a..488dd24bb 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -=============================== +================ os-client-config -=============================== +================ `os-client-config` is a library for collecting client configuration for using an OpenStack cloud in a consistent and comprehensive manner. It @@ -27,7 +27,9 @@ it by setting `OS_CLOUD_NAME`. Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type -for trove set:: +for trove set + +:: export OS_DATABASE_SERVICE_TYPE=rax:database @@ -228,6 +230,7 @@ Usage ----- The simplest and least useful thing you can do is: + :: python -m os_client_config.config @@ -236,6 +239,7 @@ Which will print out whatever if finds for your config. If you want to use it from python, which is much more likely what you want to do, things like: Get a named cloud. + :: import os_client_config @@ -245,7 +249,9 @@ Get a named cloud. print(cloud_config.name, cloud_config.region, cloud_config.config) Or, get all of the clouds. + :: + import os_client_config cloud_config = os_client_config.OpenStackConfig().get_all_clouds() From 79637b12cfebb3ab80c781b1217755832ab069c4 Mon Sep 17 00:00:00 2001 From: Simon Leinen Date: Mon, 28 Sep 2015 23:35:17 +0200 Subject: [PATCH 150/365] Added SWITCHengines vendor file Signed-off-by: Simon Leinen Change-Id: Iae386787ee2b3feecb78e3f69c7ed4e279e5acce --- doc/source/vendor-support.rst | 14 ++++++++++++++ os_client_config/vendors/switchengines.yaml | 9 +++++++++ 2 files changed, 23 insertions(+) create mode 100644 os_client_config/vendors/switchengines.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 39f488d14..598b2d496 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -247,3 +247,17 @@ nz_wlg_1 Wellington, NZ * Image API Version is 1 * Images must be in `raw` format + +switchengines +------------- + +https://keystone.cloud.switch.ch:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +LS Lausanne, CH +ZH Zurich, CH + +* Images must be in `raw` format +* Images must be uploaded using the Glance Task Interface diff --git a/os_client_config/vendors/switchengines.yaml b/os_client_config/vendors/switchengines.yaml new file mode 100644 index 000000000..ff6c50517 --- /dev/null +++ b/os_client_config/vendors/switchengines.yaml @@ -0,0 +1,9 @@ +name: switchengines +profile: + auth: + auth_url: https://keystone.cloud.switch.ch:5000/v2.0 + regions: + - LS + - ZH + image_api_use_tasks: true + image_format: raw From 723b893f22fc3db74761dc64e5296360e5739147 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 30 Sep 2015 17:27:22 -0400 Subject: [PATCH 151/365] Handle OS_CLOUD and OS_REGION_NAME friendly-like If you're using os-client-config and ansible and python-openstackclient, the envvar override support can get tricky if you've set OS_CLOUD to select a cloud for python-openstackclient, leading you to have an empty cloud called envvars when you were not actually trying to create one. Special-case pull both out, treating OS_CLOUD as a default value for get_one_cloud() and only creating an envvars cloud if OS_REGION_NAME is not the only env var provided. Change-Id: I2a5235c18f9be1e5dfc019888a0c187ef40074bc --- os_client_config/config.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index fcca30af3..2cea7eee1 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -86,7 +86,10 @@ def _get_os_environ(): and not k.startswith('OS_TEST') # infra CI var and not k.startswith('OS_STD') # infra CI var ] - if not environkeys: + # If the only environ key is region name, don't make a cloud, because + # it's being used as a cloud selector + if not environkeys or ( + len(environkeys) == 1 and 'OS_REGION_NAME' in environkeys): return None for k in environkeys: newkey = k[3:].lower() @@ -147,6 +150,9 @@ class OpenStackConfig(object): ' either your environment based cloud, or one of your' ' file-based clouds.'.format(self.config_filename, self.envvar_key)) + # Pull out OS_CLOUD so that if it's the only thing set, do not + # make an envvars cloud + self.default_cloud = os.environ.pop('OS_CLOUD', None) envvars = _get_os_environ() if envvars: @@ -523,6 +529,9 @@ class OpenStackConfig(object): on missing required auth parameters """ + if cloud is None and self.default_cloud: + cloud = self.default_cloud + if cloud is None and self.envvar_key in self.get_cloud_names(): cloud = self.envvar_key From 733c04f6d381f523991c5ccfd56b9ff722111bf0 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Wed, 30 Sep 2015 23:15:35 +0200 Subject: [PATCH 152/365] identity version is 2.0 When using the defaults values from outside, we get a not so right API version. Change-Id: I8159dd4a26b65ad242d5ba75c4a5e0dc161704fc --- os_client_config/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/defaults.yaml b/os_client_config/defaults.yaml index e81eb1649..ddb975cf0 100644 --- a/os_client_config/defaults.yaml +++ b/os_client_config/defaults.yaml @@ -6,7 +6,7 @@ disable_vendor_agent: {} dns_api_version: '2' interface: public floating_ip_source: neutron -identity_api_version: '2' +identity_api_version: '2.0' image_api_use_tasks: false image_api_version: '2' image_format: qcow2 From 7d84f102313e61aba00d7665250ac6f628f8e8f6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 1 Oct 2015 10:50:27 -0400 Subject: [PATCH 153/365] Support passing force_ipv4 to the constructor IPv6 support is detectable, so rather than having a user opt-in to it, provide a flag that can be provided to tell it that detected IPv6 support is lying. This should have to be set for far fewer people and should result in transparent opt-in to IPv6 where available. Change-Id: Ib0c4c4e8b3b7b4bcee5fa3414719969274929b9a --- README.rst | 18 ++++++----- os_client_config/cloud_config.py | 10 ++++-- os_client_config/config.py | 36 ++++++++++++++++----- os_client_config/tests/base.py | 2 +- os_client_config/tests/test_cloud_config.py | 6 ++-- os_client_config/tests/test_config.py | 24 ++++++++++---- 6 files changed, 67 insertions(+), 29 deletions(-) diff --git a/README.rst b/README.rst index 488dd24bb..21532ebdc 100644 --- a/README.rst +++ b/README.rst @@ -197,15 +197,18 @@ are connecting to OpenStack can share a cache should you desire. IPv6 ---- -IPv6 may be a thing you would prefer to use not only if the cloud supports it, -but also if your local machine support it. A simple boolean flag is settable -either in an environment variable, `OS_PREFER_IPV6`, or in the client section -of the clouds.yaml. +IPv6 is the future, and you should always use if if your cloud supports it and +if your local network supports it. Both of those are eaily detectable and all +friendly software should do the right thing. However, sometimes you might +exist in a location where you have an IPv6 stack, but something evil has +caused it to not actually function. In that case, there is a config option +you can set to unbreak you `force_ipv4`, or `OS_FORCE_IPV4` boolean +environment variable. :: client: - prefer_ipv6: true + force_ipv4: true clouds: mordred: profile: hp @@ -222,9 +225,8 @@ of the clouds.yaml. project_name: mordred@inaugust.com region_name: DFW -The above snippet will tell client programs to prefer returning an IPv6 -address. This will result in calls to, for instance, `shade`'s `get_public_ip` -to return an IPv4 address on HP, and an IPv6 address on Rackspace. +The above snippet will tell client programs to prefer returning an IPv4 +address. Usage ----- diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 86b4f5073..c8b5269ac 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -17,11 +17,11 @@ import warnings class CloudConfig(object): def __init__(self, name, region, config, - prefer_ipv6=False, auth_plugin=None): + force_ipv4=False, auth_plugin=None): self.name = name self.region = region self.config = config - self._prefer_ipv6 = prefer_ipv6 + self._force_ipv4 = force_ipv4 self._auth = auth_plugin def __getattr__(self, key): @@ -107,7 +107,11 @@ class CloudConfig(object): @property def prefer_ipv6(self): - return self._prefer_ipv6 + return not self._force_ipv4 + + @property + def force_ipv4(self): + return self._force_ipv4 def get_auth(self): """Return a keystoneauth plugin from the auth credentials.""" diff --git a/os_client_config/config.py b/os_client_config/config.py index 2cea7eee1..f43fe7458 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -113,7 +113,7 @@ def _auth_update(old_dict, new_dict): class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, - override_defaults=None): + override_defaults=None, force_ipv4=None): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES @@ -135,11 +135,28 @@ class OpenStackConfig(object): # Grab ipv6 preference settings from env client_config = self.cloud_config.get('client', {}) - self.prefer_ipv6 = get_boolean( - os.environ.pop( - 'OS_PREFER_IPV6', client_config.get( - 'prefer_ipv6', client_config.get( - 'prefer-ipv6', False)))) + + if force_ipv4 is not None: + # If it's passed in to the constructor, honor it. + self.force_ipv4 = force_ipv4 + else: + # Get the backwards compat value + prefer_ipv6 = get_boolean( + os.environ.pop( + 'OS_PREFER_IPV6', client_config.get( + 'prefer_ipv6', client_config.get( + 'prefer-ipv6', True)))) + force_ipv4 = get_boolean( + os.environ.pop( + 'OS_FORCE_IPV4', client_config.get( + 'force_ipv4', client_config.get( + 'broken-ipv6', False)))) + + self.force_ipv4 = force_ipv4 + if not prefer_ipv6: + # this will only be false if someone set it explicitly + # honor their wishes + self.force_ipv4 = True # Next, process environment variables and add them to the mix self.envvar_key = os.environ.pop('OS_CLOUD_NAME', 'envvars') @@ -586,7 +603,10 @@ class OpenStackConfig(object): if hasattr(value, 'format'): config[key] = value.format(**config) - prefer_ipv6 = config.pop('prefer_ipv6', self.prefer_ipv6) + force_ipv4 = config.pop('force_ipv4', self.force_ipv4) + prefer_ipv6 = config.pop('prefer_ipv6', True) + if not prefer_ipv6: + force_ipv4 = True if cloud is None: cloud_name = '' @@ -595,7 +615,7 @@ class OpenStackConfig(object): return cloud_config.CloudConfig( name=cloud_name, region=config['region_name'], config=self._normalize_keys(config), - prefer_ipv6=prefer_ipv6, + force_ipv4=force_ipv4, auth_plugin=auth_plugin) @staticmethod diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index cbf58da36..36c3bfb27 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -40,7 +40,7 @@ VENDOR_CONF = { } USER_CONF = { 'client': { - 'prefer_ipv6': True, + 'force_ipv4': True, }, 'clouds': { '_test-cloud_': { diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 4f5260fc1..c9317ad00 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -50,7 +50,7 @@ class TestCloudConfig(base.TestCase): self.assertIsNone(cc.x) # Test default ipv6 - self.assertFalse(cc.prefer_ipv6) + self.assertFalse(cc.force_ipv4) def test_iteration(self): cc = cloud_config.CloudConfig("test1", "region-al", fake_config_dict) @@ -116,8 +116,8 @@ class TestCloudConfig(base.TestCase): def test_ipv6(self): cc = cloud_config.CloudConfig( - "test1", "region-al", fake_config_dict, prefer_ipv6=True) - self.assertTrue(cc.prefer_ipv6) + "test1", "region-al", fake_config_dict, force_ipv4=True) + self.assertTrue(cc.force_ipv4) def test_getters(self): cc = cloud_config.CloudConfig("test1", "region-al", fake_services_dict) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 3331b3374..271c40be8 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -136,17 +136,29 @@ class TestConfig(base.TestCase): c.get_one_cloud(cloud='defaults', validate=False) def test_prefer_ipv6_true(self): - c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud(cloud='_test-cloud_') - self.assertTrue(cc.prefer_ipv6) - - def test_prefer_ipv6_false(self): c = config.OpenStackConfig(config_files=[self.no_yaml], vendor_files=[self.no_yaml]) cc = c.get_one_cloud(cloud='defaults', validate=False) + self.assertTrue(cc.prefer_ipv6) + + def test_prefer_ipv6_false(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud(cloud='_test-cloud_') self.assertFalse(cc.prefer_ipv6) + def test_force_ipv4_true(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud(cloud='_test-cloud_') + self.assertTrue(cc.force_ipv4) + + def test_force_ipv4_false(self): + c = config.OpenStackConfig(config_files=[self.no_yaml], + vendor_files=[self.no_yaml]) + cc = c.get_one_cloud(cloud='defaults', validate=False) + self.assertFalse(cc.force_ipv4) + def test_get_one_cloud_auth_merge(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml]) cc = c.get_one_cloud(cloud='_test-cloud_', auth={'username': 'user'}) From 21ff307d130e758242a698312d6a14fc0d5b620e Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 1 Oct 2015 13:37:13 -0400 Subject: [PATCH 154/365] Put in override for Rackspace broken neutron Rackspace puts neutron in the service catalog but it does not work (404s the endpoint) While it's easy enough to try/except around network calls, it's also a known fact and can be communicated clearly. Allow things that want to short-circuit read the value from the config and react accordingly. Note: This is not a value going in to the defaults, because the fact that this is happening at all is highly strange and hopefully something that can be solved by collaborating on future iterations of DefCore. However, communicating clearly that it's a known issue is helpful for a class of users. Change-Id: I506f4dd397608f2feb6fbda48d297d283312a169 --- os_client_config/vendors/rackspace.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/os_client_config/vendors/rackspace.yaml b/os_client_config/vendors/rackspace.yaml index 5cf7a44e6..c37802976 100644 --- a/os_client_config/vendors/rackspace.yaml +++ b/os_client_config/vendors/rackspace.yaml @@ -17,3 +17,4 @@ profile: disable_vendor_agent: vm_mode: hvm xenapi_use_agent: false + has_network: false From d3c82ab42835c2bfe9817d2f8a571490153879e1 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 1 Oct 2015 14:52:18 -0400 Subject: [PATCH 155/365] Fix two typos Change-Id: Idf3df94ea039f07fa958a3afadf3388221ffa2ff --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 21532ebdc..7b737b9de 100644 --- a/README.rst +++ b/README.rst @@ -197,8 +197,8 @@ are connecting to OpenStack can share a cache should you desire. IPv6 ---- -IPv6 is the future, and you should always use if if your cloud supports it and -if your local network supports it. Both of those are eaily detectable and all +IPv6 is the future, and you should always use it if your cloud supports it and +if your local network supports it. Both of those are easily detectable and all friendly software should do the right thing. However, sometimes you might exist in a location where you have an IPv6 stack, but something evil has caused it to not actually function. In that case, there is a config option From a9d5827a0929d5de1fc0951412f96bc031eb2409 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Fri, 2 Oct 2015 12:11:50 +0200 Subject: [PATCH 156/365] Some cleanup Remove identity api version 2 support (default) and some cleanup in the doc (tables properly formated) Change-Id: I5d90723dd54905c8a67c52c6111bdf089f4346a0 --- doc/source/vendor-support.rst | 2 ++ os_client_config/vendors/catalyst.yaml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 598b2d496..b61a05b8c 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -244,6 +244,7 @@ Region Name Human Name ============== ================ nz-por-1 Porirua, NZ nz_wlg_1 Wellington, NZ +============== ================ * Image API Version is 1 * Images must be in `raw` format @@ -258,6 +259,7 @@ Region Name Human Name ============== ================ LS Lausanne, CH ZH Zurich, CH +============== ================ * Images must be in `raw` format * Images must be uploaded using the Glance Task Interface diff --git a/os_client_config/vendors/catalyst.yaml b/os_client_config/vendors/catalyst.yaml index 348ceafc8..14cdf254f 100644 --- a/os_client_config/vendors/catalyst.yaml +++ b/os_client_config/vendors/catalyst.yaml @@ -5,6 +5,5 @@ profile: regions: - nz-por-1 - nz_wlg_1 - identity_api_version: '2' image_api_version: '1' image_format: raw From 379962f0e8289e2db85a0eeb7b51d10cff70e67d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 2 Oct 2015 11:08:22 -0400 Subject: [PATCH 157/365] Add universal=1 to setup.cfg to build python 3 wheels os-client-config gates on python3. Change-Id: I887ab2f2f2436e7423eab8abc23655423ee7b226 --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index df3434f0f..bc4f128cf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,3 +31,6 @@ all_files = 1 [upload_sphinx] upload-dir = doc/build/html + +[wheel] +universal = 1 From 3609cd59078873bc44ebbe7641889423c9f2b8cd Mon Sep 17 00:00:00 2001 From: Anita Kuno Date: Tue, 6 Oct 2015 17:18:54 -0400 Subject: [PATCH 158/365] Adds some lines to complete table formatting Currently the vendor support list is mis-formatted for the last two entries on the page. It appears that the tables don't have lines underneath them completing the formatting. This patch adds lines under the last two vendor entries fixing the formatting of the tables for those entries. Change-Id: I6327eef99059291fcce5c9493a622bec33da3824 --- doc/source/vendor-support.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 598b2d496..b61a05b8c 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -244,6 +244,7 @@ Region Name Human Name ============== ================ nz-por-1 Porirua, NZ nz_wlg_1 Wellington, NZ +============== ================ * Image API Version is 1 * Images must be in `raw` format @@ -258,6 +259,7 @@ Region Name Human Name ============== ================ LS Lausanne, CH ZH Zurich, CH +============== ================ * Images must be in `raw` format * Images must be uploaded using the Glance Task Interface From 801d09a8dae6a976ddf1603a0b4ceb1bd4166731 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Fri, 9 Oct 2015 15:18:27 -0500 Subject: [PATCH 159/365] Add auth hook for OpenStackClient Since os_client_config started pulling in ksa plugins we have some duplicate behaviour with OSC. If o-c-c is going to be loading auth plugins OSC needs a way to hook into the notch between setting config values and actually loading the plugins in order to maintain existing behaviours for compatibility. It may also be desirable at some point to defer the plugin loading, OSC did that orignally due to ksc's poor load times, ksa shouldn't have the same problem. Change-Id: I1e8a50e5467fd4f9151a64aa7ae140d2654f3186 --- os_client_config/config.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index f43fe7458..c87870d2a 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -433,6 +433,14 @@ class OpenStackConfig(object): if d_opt_name in config: return config[d_opt_name] + def auth_config_hook(self, config): + """Allow examination of config values before loading auth plugin + + OpenStackClient will override this to perform additional chacks + on auth_type. + """ + return config + def _get_auth_loader(self, config): # Re-use the admin_token plugin for the "None" plugin # since it does not look up endpoints or tokens but rather @@ -576,6 +584,11 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) + # NOTE(dtroyer): OSC needs a hook into the auth args before the + # plugin is loaded in order to maintain backward- + # compatible behaviour + config = self.auth_config_hook(config) + if loading: if validate: try: From 6113d037f6240666e3c958db6e83a242e7c8531d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 14 Oct 2015 10:18:59 -0400 Subject: [PATCH 160/365] Pass OpenStackConfig in to CloudConfig for caches The unit of consumption is the CloudConfig object, but the cache settings are found only globally in the OpenStackConfig object. Pass the OpenStackConfig object in so that a user consuming settings from a CloudConfig can find out what the cache settings are. Change-Id: I633e35b1d7f295581d7abed9c3957e802690614e --- os_client_config/cloud_config.py | 20 +++++++++++++++++++- os_client_config/config.py | 7 ++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index c8b5269ac..f0c414387 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -17,12 +17,14 @@ import warnings class CloudConfig(object): def __init__(self, name, region, config, - force_ipv4=False, auth_plugin=None): + force_ipv4=False, auth_plugin=None, + openstack_config=None): self.name = name self.region = region self.config = config self._force_ipv4 = force_ipv4 self._auth = auth_plugin + self._openstack_config = openstack_config def __getattr__(self, key): """Return arbitrary attributes.""" @@ -116,3 +118,19 @@ class CloudConfig(object): def get_auth(self): """Return a keystoneauth plugin from the auth credentials.""" return self._auth + + def get_cache_interval(self): + if self._openstack_config: + return self._openstack_config.get_cache_interval() + + def get_cache_path(self): + if self._openstack_config: + return self._openstack_config.get_cache_path() + + def get_cache_class(self): + if self._openstack_config: + return self._openstack_config.get_cache_class() + + def get_cache_arguments(self): + if self._openstack_config: + return self._openstack_config.get_cache_arguments() diff --git a/os_client_config/config.py b/os_client_config/config.py index c87870d2a..6d37835ed 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -221,6 +221,9 @@ class OpenStackConfig(object): new_config[key] = value return new_config + def get_cache_interval(self): + return self._cache_max_age + def get_cache_max_age(self): return self._cache_max_age @@ -629,7 +632,9 @@ class OpenStackConfig(object): name=cloud_name, region=config['region_name'], config=self._normalize_keys(config), force_ipv4=force_ipv4, - auth_plugin=auth_plugin) + auth_plugin=auth_plugin, + openstack_config=self + ) @staticmethod def set_one_cloud(config_file, cloud, set_config=None): From ec4ebbdd277c2125734640f2b3cd1184c4c6bf86 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 14 Oct 2015 09:27:21 -0700 Subject: [PATCH 161/365] Add an API reference to the docs Change-Id: I2c40965378dedd67808875eb9453ab8664f0fee8 --- doc/source/api-reference.rst | 10 ++++++++++ doc/source/index.rst | 1 + 2 files changed, 11 insertions(+) create mode 100644 doc/source/api-reference.rst diff --git a/doc/source/api-reference.rst b/doc/source/api-reference.rst new file mode 100644 index 000000000..dfa7f31cb --- /dev/null +++ b/doc/source/api-reference.rst @@ -0,0 +1,10 @@ +============= +API Reference +============= + +.. module:: os_client_config + :synopsis: OpenStack client configuration + +.. autoclass:: os_client_config.OpenStackConfig + :members: + :inherited-members: diff --git a/doc/source/index.rst b/doc/source/index.rst index 0f793a1a3..cc5dbf470 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -6,6 +6,7 @@ vendor-support contributing installation + api-reference Indices and tables ================== From f6681a83192386fcf27479c820998691a43e8b79 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 14 Oct 2015 12:32:20 -0400 Subject: [PATCH 162/365] Fix documentation around regions It turns out region_name is an important parameter - it's not just another kwarg that will get passed through. Change-Id: I5cca8d324a1dcd1355991df793fe29eedfa15dc0 --- README.rst | 2 +- os_client_config/config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7b737b9de..b0466f797 100644 --- a/README.rst +++ b/README.rst @@ -247,7 +247,7 @@ Get a named cloud. import os_client_config cloud_config = os_client_config.OpenStackConfig().get_one_cloud( - 'hp', 'region-b.geo-1') + 'hp', region_name='region-b.geo-1') print(cloud_config.name, cloud_config.region, cloud_config.config) Or, get all of the clouds. diff --git a/os_client_config/config.py b/os_client_config/config.py index 6d37835ed..bff2f5cce 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -551,6 +551,7 @@ class OpenStackConfig(object): An argparse Namespace object; allows direct passing in of argparse options to be added to the cloud config. Values of None and '' will be removed. + :param region_name: Name of the region of the cloud. :param kwargs: Additional configuration options :raises: keystoneauth1.exceptions.MissingRequiredOptions From 512ca01715683f7c46985677e7b9be8e3d7a191c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 14 Oct 2015 12:40:50 -0400 Subject: [PATCH 163/365] Validate requested region against region list We have lists of valid regions for clouds, but specifying a region incorrectly is a common mistake (specifying dfw instead of DFW, for instance) Throw an error early with a helpful error message. Change-Id: I55edf20c6dddde4a4d71dad33a41d4e10448ddac --- os_client_config/config.py | 10 +++++++++- os_client_config/tests/test_config.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index bff2f5cce..b2c164936 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -573,7 +573,15 @@ class OpenStackConfig(object): # Regions is a list that we can use to create a list of cloud/region # objects. It does not belong in the single-cloud dict - config.pop('regions', None) + regions = config.pop('regions', None) + if regions and args['region_name'] not in regions: + raise exceptions.OpenStackConfigException( + 'Region {region_name} is not a valid region name for cloud' + ' {cloud}. Valid choices are {region_list}. Please note that' + ' region names are case sensitive.'.format( + region_name=args['region_name'], + region_list=','.join(regions), + cloud=cloud)) # Can't just do update, because None values take over for (key, val) in iter(args.items()): diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 271c40be8..01307488a 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -298,6 +298,23 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(cc.region_name, 'region1') self.assertIsNone(cc.snack_type) + def test_get_one_cloud_bad_region(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + self.assertRaises( + exceptions.OpenStackConfigException, + c.get_one_cloud, + cloud='_test_cloud_regions', region_name='bad') + + def test_get_one_cloud_bad_region_no_regions(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud(cloud='_test-cloud_', region_name='bad_region') + self._assert_cloud_details(cc) + self.assertEqual(cc.region_name, 'bad_region') + def test_get_one_cloud_no_argparse_region2(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From b06adf563e42798d02ee9153ccb9096fda3bac92 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 16 Oct 2015 11:13:13 -0400 Subject: [PATCH 164/365] Add Rackspace LON region Change-Id: I01df6e7f0cf09d52a50e1af14cdc76db90b44320 --- doc/source/vendor-support.rst | 7 ++++--- os_client_config/vendors/rackspace.yaml | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index b61a05b8c..5519928a4 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -49,10 +49,11 @@ https://identity.api.rackspacecloud.com/v2.0/ Region Name Human Name ============== ================ DFW Dallas -ORD Chicago -IAD Washington, D.C. -SYD Sydney HKG Hong Kong +IAD Washington, D.C. +LON London +ORD Chicago +SYD Sydney ============== ================ * Database Service Type is `rax:database` diff --git a/os_client_config/vendors/rackspace.yaml b/os_client_config/vendors/rackspace.yaml index c37802976..a28d49366 100644 --- a/os_client_config/vendors/rackspace.yaml +++ b/os_client_config/vendors/rackspace.yaml @@ -8,6 +8,7 @@ profile: - IAD - ORD - SYD + - LON database_service_type: rax:database compute_service_name: cloudServersOpenStack image_api_use_tasks: true From 790fac98542ba303274180831cbd9a03604a84e8 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 16 Oct 2015 16:04:21 -0400 Subject: [PATCH 165/365] Clean up cache interface, add support for services We just added an unreleased interface method to the CloudConfig object - but maybe that should have been more aligned with dogpile words. SO - change the docs to reference the dogpile words and add support for that, while keeping backwards compat support for people using max_age. Also, do the -/_ transform on the cache config like elsewhere. Then, while we're in there, add support for per-service cache timings. We need this in nodepool and shade is adding support, so ability to configure it will be important. Change-Id: I31190a31ab0b79fc080db3611c0cd584076387d4 --- README.rst | 13 +++++++++-- os_client_config/cloud_config.py | 21 +++++++++++++++-- os_client_config/config.py | 39 ++++++++++++++++++++++++-------- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index b0466f797..f16bbc091 100644 --- a/README.rst +++ b/README.rst @@ -168,9 +168,15 @@ understands passing through cache settings to dogpile.cache, with the following behaviors: * Listing no config settings means you get a null cache. -* `cache.max_age` and nothing else gets you memory cache. +* `cache.expiration_time` and nothing else gets you memory cache. * Otherwise, `cache.class` and `cache.arguments` are passed in +Different cloud behaviors are also differently expensive to deal with. If you +want to get really crazy and tweak stuff, you can specify different expiration +times on a per-resource basis by passing values, in seconds to an expiration +mapping keyed on the singular name of the resource. A value of `-1` indicates +that the resource should never expire. + `os-client-config` does not actually cache anything itself, but it collects and presents the cache information so that your various applications that are connecting to OpenStack can share a cache should you desire. @@ -179,10 +185,13 @@ are connecting to OpenStack can share a cache should you desire. cache: class: dogpile.cache.pylibmc - max_age: 3600 + expiration_time: 3600 arguments: url: - 127.0.0.1 + expiration: + server: 5 + flavor: -1 clouds: mordred: profile: hp diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index f0c414387..63ff8d29b 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -119,9 +119,9 @@ class CloudConfig(object): """Return a keystoneauth plugin from the auth credentials.""" return self._auth - def get_cache_interval(self): + def get_cache_expiration_time(self): if self._openstack_config: - return self._openstack_config.get_cache_interval() + return self._openstack_config.get_cache_expiration_time() def get_cache_path(self): if self._openstack_config: @@ -134,3 +134,20 @@ class CloudConfig(object): def get_cache_arguments(self): if self._openstack_config: return self._openstack_config.get_cache_arguments() + + def get_cache_expiration(self): + if self._openstack_config: + return self._openstack_config.get_cache_expiration() + + def get_cache_resource_expiration(self, resource): + """Get expiration time for a resource + + :param resource: Name of the resource type + + :returns: Expiration time for the resource type or None + """ + if self._openstack_config: + expiration = self._openstack_config.get_cache_expiration() + if resource not in expiration: + return None + return expiration[resource] diff --git a/os_client_config/config.py b/os_client_config/config.py index b2c164936..2e3644bd0 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -182,21 +182,34 @@ class OpenStackConfig(object): self.cloud_config = dict( clouds=dict(defaults=dict(self.defaults))) - self._cache_max_age = 0 + self._cache_expiration_time = 0 self._cache_path = CACHE_PATH self._cache_class = 'dogpile.cache.null' self._cache_arguments = {} + self._cache_expiration = {} if 'cache' in self.cloud_config: - self._cache_max_age = self.cloud_config['cache'].get( - 'max_age', self._cache_max_age) - if self._cache_max_age: + cache_settings = self._normalize_keys(self.cloud_config['cache']) + + # expiration_time used to be 'max_age' but the dogpile setting + # is expiration_time. Support max_age for backwards compat. + self._cache_expiration_time = cache_settings.get( + 'expiration_time', cache_settings.get( + 'max_age', self._cache_expiration_time)) + + # If cache class is given, use that. If not, but if cache time + # is given, default to memory. Otherwise, default to nothing. + # to memory. + if self._cache_expiration_time: self._cache_class = 'dogpile.cache.memory' - self._cache_path = os.path.expanduser( - self.cloud_config['cache'].get('path', self._cache_path)) self._cache_class = self.cloud_config['cache'].get( 'class', self._cache_class) - self._cache_arguments = self.cloud_config['cache'].get( + + self._cache_path = os.path.expanduser( + cache_settings.get('path', self._cache_path)) + self._cache_arguments = cache_settings.get( 'arguments', self._cache_arguments) + self._cache_expiration = cache_settings.get( + 'expiration', self._cache_expiration) def _load_config_file(self): return self._load_yaml_file(self._config_files) @@ -221,11 +234,14 @@ class OpenStackConfig(object): new_config[key] = value return new_config + def get_cache_expiration_time(self): + return self._cache_expiration_time + def get_cache_interval(self): - return self._cache_max_age + return self._cache_expiration_time def get_cache_max_age(self): - return self._cache_max_age + return self._cache_expiration_time def get_cache_path(self): return self._cache_path @@ -234,7 +250,10 @@ class OpenStackConfig(object): return self._cache_class def get_cache_arguments(self): - return self._cache_arguments + return self._cache_arguments.copy() + + def get_cache_expiration(self): + return self._cache_expiration.copy() def _get_regions(self, cloud): if cloud not in self.cloud_config['clouds']: From f39cb4b9330307eb363b2e1707f87c9d4d5e8539 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 26 Oct 2015 13:36:13 +0900 Subject: [PATCH 166/365] Always pull regions from vendor profiles Regions are special, since they're a driver for creating a cloud config object in the first place. We weren't syncing them from the vendor profile, so if the region wasn't set in the clouds.yaml, region name validation (and get_all_clouds) would fail. Change-Id: I5b8d6807a86b87a7f69f523f2fee2784389fab0a --- os_client_config/config.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 2e3644bd0..aa161e474 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -270,7 +270,17 @@ class OpenStackConfig(object): " parameter in {0} instead.".format(self.config_filename)) return regions else: - return [''] + # crappit. we don't have a region defined. + new_cloud = dict() + our_cloud = self.cloud_config['clouds'].get(cloud, dict()) + self._expand_vendor_profile(cloud, new_cloud, our_cloud) + if 'regions' in new_cloud and new_cloud['regions']: + return new_cloud['regions'] + elif 'region_name' in new_cloud and new_cloud['region_name']: + return [new_cloud['region_name']] + else: + # Wow. We really tried + return [''] def _get_region(self, cloud=None): return self._get_regions(cloud)[0] @@ -291,7 +301,18 @@ class OpenStackConfig(object): # Get the defaults cloud.update(self.defaults) + self._expand_vendor_profile(name, cloud, our_cloud) + if 'auth' not in cloud: + cloud['auth'] = dict() + + _auth_update(cloud, our_cloud) + if 'cloud' in cloud: + del cloud['cloud'] + + return self._fix_backwards_madness(cloud) + + def _expand_vendor_profile(self, name, cloud, our_cloud): # Expand a profile if it exists. 'cloud' is an old confusing name # for this. profile_name = our_cloud.get('profile', our_cloud.get('cloud', None)) @@ -314,15 +335,6 @@ class OpenStackConfig(object): " the cloud '{1}'".format(profile_name, name)) - if 'auth' not in cloud: - cloud['auth'] = dict() - - _auth_update(cloud, our_cloud) - if 'cloud' in cloud: - del cloud['cloud'] - - return self._fix_backwards_madness(cloud) - def _fix_backwards_madness(self, cloud): cloud = self._fix_backwards_project(cloud) cloud = self._fix_backwards_auth_plugin(cloud) From f8804bd4f5f9ab5a1e83ec5decfaa8474f1258b6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 27 Oct 2015 07:27:09 +0900 Subject: [PATCH 167/365] Use assertDictEqual to test dict equality Just using equals is not a consistent test, because dict ordering can change. assertDictEqual tests that the contents of the two dicts are the same. Change-Id: I1b084a3fee91d73d57a6b9e6a2d0ab9c186e5572 --- os_client_config/tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 01307488a..d225b7c17 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -380,4 +380,4 @@ class TestBackwardsCompatibility(base.TestCase): 'interface': 'public', 'auth_type': 'v3password', } - self.assertEqual(expected, result) + self.assertDictEqual(expected, result) From 4ae04265ffc015259a2c3709b03edcfe86953833 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 26 Oct 2015 14:11:55 +0900 Subject: [PATCH 168/365] Allow for templated variables in auth_url Some clouds have auth_urls per region, which means putting a static auth_url in the config file isn't going to work. Allow {} substitutions in the auth_url so that other elements of the auth dict can be injected. This will not solve _everything, but it will solve all of the currently identified issues, and should be upwardly open if we need something more complex later. Change-Id: I6107d0669734647cfa0bef118bdf7739949991ad --- os_client_config/config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 2e3644bd0..578e5cd8f 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -615,6 +615,13 @@ class OpenStackConfig(object): if type(config[key]) is not bool: config[key] = get_boolean(config[key]) + # TODO(mordred): Special casing auth_url here. We should + # come back to this betterer later so that it's + # more generalized + if 'auth' in config and 'auth_url' in config['auth']: + config['auth']['auth_url'] = config['auth']['auth_url'].format( + **config) + # NOTE(dtroyer): OSC needs a hook into the auth args before the # plugin is loaded in order to maintain backward- # compatible behaviour From c71ba514a3827f09312e873899a39d4974bd1e4c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 26 Oct 2015 14:16:53 +0900 Subject: [PATCH 169/365] Add conoha public cloud Change-Id: I05a716bc54c98390a7cbeb352338a7c6cd7e86c3 --- doc/source/vendor-support.rst | 15 +++++++++++++++ os_client_config/vendors/conoha.yaml | 8 ++++++++ 2 files changed, 23 insertions(+) create mode 100644 os_client_config/vendors/conoha.yaml diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 5519928a4..21995315e 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -264,3 +264,18 @@ ZH Zurich, CH * Images must be in `raw` format * Images must be uploaded using the Glance Task Interface + +conoha +------ + +https://identity.%(region_name)s.conoha.io/v2.0 + +============== ================ +Region Name Human Name +============== ================ +tyo1 Tokyo, JP +sin1 Singapore +lon1 London, UK +============== ================ + +* Images cannot be uploaded diff --git a/os_client_config/vendors/conoha.yaml b/os_client_config/vendors/conoha.yaml new file mode 100644 index 000000000..1ed4063ec --- /dev/null +++ b/os_client_config/vendors/conoha.yaml @@ -0,0 +1,8 @@ +name: conoha +profile: + auth: + auth_url: https://identity.{region_name}.conoha.io/v2.0 + regions: + - sin1 + - lon1 + - tyo1 From 2aaba84c7c4dd71420b7881358f89ec1e061c11c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 27 Oct 2015 07:08:49 +0900 Subject: [PATCH 170/365] Sort vendor list We've got enough of them that the arbitrary sort order was a bit weird. Sort alphabetically. Change-Id: I6d4ec563f03d1b25bad7d8337db90007a562e97c --- doc/source/vendor-support.rst | 350 +++++++++++++++++----------------- 1 file changed, 175 insertions(+), 175 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 21995315e..d082ba04f 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -24,6 +24,121 @@ These are the default behaviors unless a cloud is configured differently. * Security groups are provided by Neutron * Vendor specific agents are not used +auro +---- + +https://api.auro.io:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +van1 Vancouver, BC +============== ================ + +* Public IPv4 is provided via NAT with Nova Floating IP +* Floating IPs are provided by Nova +* Security groups are provided by Nova + +catalyst +-------- + +https://api.cloud.catalyst.net.nz:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +nz-por-1 Porirua, NZ +nz_wlg_1 Wellington, NZ +============== ================ + +* Image API Version is 1 +* Images must be in `raw` format + +citycloud +--------- + +https://identity1.citycloud.com:5000/v3/ + +============== ================ +Region Name Human Name +============== ================ +Lon1 London, UK +Sto2 Stockholm, SE +Kna1 Karlskrona, SE +============== ================ + +* Identity API Version is 3 +* Public IPv4 is provided via NAT with Neutron Floating IP + +conoha +------ + +https://identity.%(region_name)s.conoha.io/v2.0 + +============== ================ +Region Name Human Name +============== ================ +tyo1 Tokyo, JP +sin1 Singapore +lon1 London, UK +============== ================ + +* Images cannot be uploaded + +datacentred +----------- + +https://compute.datacentred.io:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +sal01 Manchester, UK +============== ================ + +* Image API Version is 1 + +dreamhost +--------- + +https://keystone.dream.io/v2.0 + +============== ================ +Region Name Human Name +============== ================ +RegionOne Region One +============== ================ + +* Images must be in `raw` format +* Public IPv4 is provided via NAT with Neutron Floating IP +* IPv6 is provided to every server + +elastx +------ + +https://ops.elastx.net:5000/v2.0 + +============== ================ +Region Name Human Name +============== ================ +regionOne Region One +============== ================ + +* Public IPv4 is provided via NAT with Neutron Floating IP + +entercloudsuite +--------------- + +https://api.entercloudsuite.com/v2.0 + +============== ================ +Region Name Human Name +============== ================ +nl-ams1 Amsterdam, NL +it-mil1 Milan, IT +de-fra1 Frankfurt, DE +============== ================ + hp -- @@ -40,6 +155,37 @@ region-b.geo-1 US East * Image API Version is 1 * Public IPv4 is provided via NAT with Neutron Floating IP +internap +-------- + +https://identity.api.cloud.iweb.com/v2.0 + +============== ================ +Region Name Human Name +============== ================ +ams01 Amsterdam, NL +da01 Dallas, TX +nyj01 New York, NY +============== ================ + +* Image API Version is 1 +* Floating IPs are not supported + +ovh +--- + +https://auth.cloud.ovh.net/v2.0 + +============== ================ +Region Name Human Name +============== ================ +SBG1 Strassbourg, FR +GRA1 Gravelines, FR +============== ================ + +* Images must be in `raw` format +* Floating IPs are not supported + rackspace --------- @@ -68,32 +214,6 @@ SYD Sydney :vm_mode: hvm :xenapi_use_agent: False -dreamhost ---------- - -https://keystone.dream.io/v2.0 - -============== ================ -Region Name Human Name -============== ================ -RegionOne Region One -============== ================ - -* Images must be in `raw` format -* Public IPv4 is provided via NAT with Neutron Floating IP -* IPv6 is provided to every server - -vexxhost --------- - -http://auth.api.thenebulacloud.com:5000/v2.0/ - -============== ================ -Region Name Human Name -============== ================ -ca-ymq-1 Montreal -============== ================ - runabove -------- @@ -108,148 +228,6 @@ BHS-1 Beauharnois, QC * Floating IPs are not supported -unitedstack ------------ - -https://identity.api.ustack.com/v3 - -============== ================ -Region Name Human Name -============== ================ -bj1 Beijing -gd1 Guangdong -============== ================ - -* Identity API Version is 3 -* Images must be in `raw` format - -auro ----- - -https://api.auro.io:5000/v2.0 - -============== ================ -Region Name Human Name -============== ================ -van1 Vancouver, BC -============== ================ - -* Public IPv4 is provided via NAT with Nova Floating IP -* Floating IPs are provided by Nova -* Security groups are provided by Nova - -ovh ---- - -https://auth.cloud.ovh.net/v2.0 - -============== ================ -Region Name Human Name -============== ================ -SBG1 Strassbourg, FR -GRA1 Gravelines, FR -============== ================ - -* Images must be in `raw` format -* Floating IPs are not supported - -citycloud ---------- - -https://identity1.citycloud.com:5000/v3/ - -============== ================ -Region Name Human Name -============== ================ -Lon1 London, UK -Sto2 Stockholm, SE -Kna1 Karlskrona, SE -============== ================ - -* Identity API Version is 3 -* Public IPv4 is provided via NAT with Neutron Floating IP - -elastx ------- - -https://ops.elastx.net:5000/v2.0 - -============== ================ -Region Name Human Name -============== ================ -regionOne Region One -============== ================ - -* Public IPv4 is provided via NAT with Neutron Floating IP - -entercloudsuite ---------------- - -https://api.entercloudsuite.com/v2.0 - -============== ================ -Region Name Human Name -============== ================ -nl-ams1 Amsterdam, NL -it-mil1 Milan, IT -de-fra1 Frankfurt, DE -============== ================ - -ultimum -------- - -https://console.ultimum-cloud.com:5000/v2.0 - -============== ================ -Region Name Human Name -============== ================ -RegionOne Region One -============== ================ - -datacentred ------------ - -https://compute.datacentred.io:5000/v2.0 - -============== ================ -Region Name Human Name -============== ================ -sal01 Manchester, UK -============== ================ - -* Image API Version is 1 - -internap --------- - -https://identity.api.cloud.iweb.com/v2.0 - -============== ================ -Region Name Human Name -============== ================ -ams01 Amsterdam, NL -da01 Dallas, TX -nyj01 New York, NY -============== ================ - -* Image API Version is 1 -* Floating IPs are not supported - -catalyst --------- - -https://api.cloud.catalyst.net.nz:5000/v2.0 - -============== ================ -Region Name Human Name -============== ================ -nz-por-1 Porirua, NZ -nz_wlg_1 Wellington, NZ -============== ================ - -* Image API Version is 1 -* Images must be in `raw` format - switchengines ------------- @@ -265,17 +243,39 @@ ZH Zurich, CH * Images must be in `raw` format * Images must be uploaded using the Glance Task Interface -conoha ------- +ultimum +------- -https://identity.%(region_name)s.conoha.io/v2.0 +https://console.ultimum-cloud.com:5000/v2.0 ============== ================ Region Name Human Name ============== ================ -tyo1 Tokyo, JP -sin1 Singapore -lon1 London, UK +RegionOne Region One ============== ================ -* Images cannot be uploaded +unitedstack +----------- + +https://identity.api.ustack.com/v3 + +============== ================ +Region Name Human Name +============== ================ +bj1 Beijing +gd1 Guangdong +============== ================ + +* Identity API Version is 3 +* Images must be in `raw` format + +vexxhost +-------- + +http://auth.api.thenebulacloud.com:5000/v2.0/ + +============== ================ +Region Name Human Name +============== ================ +ca-ymq-1 Montreal +============== ================ From 0cee2505032bbbb5d976d00c3fc096ec1eb6c679 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 27 Oct 2015 07:11:50 +0900 Subject: [PATCH 171/365] Aligned a few words in the docs We say "not supported" when the cloud doens't do something, but there were two places where we used other words. Fix them. Also, make the image properties a blockquote. Change-Id: Id933f7d38d706c6fb3ed9f59ad10c2e1e2d73c43 --- doc/source/vendor-support.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index d082ba04f..3b4a37078 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -83,7 +83,7 @@ sin1 Singapore lon1 London, UK ============== ================ -* Images cannot be uploaded +* Image upload is not supported datacentred ----------- @@ -206,13 +206,13 @@ SYD Sydney * Compute Service Name is `cloudServersOpenStack` * Images must be in `vhd` format * Images must be uploaded using the Glance Task Interface -* Floating IPs are not needed +* Floating IPs are not supported * Public IPv4 is directly routable via static config by Nova * IPv6 is provided to every server * Security groups are not supported -* Uploaded Images need properties to not use vendor agent -:vm_mode: hvm -:xenapi_use_agent: False +* Uploaded Images need properties to not use vendor agent:: + :vm_mode: hvm + :xenapi_use_agent: False runabove -------- From eba0221a26f49f7b4c807f4732846ef2d42db169 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Tue, 3 Nov 2015 12:39:57 +1300 Subject: [PATCH 172/365] Fix typo in Catalyst region configs Small typo making one region not useful for customers (my bad!). Change-Id: If2680b521f92789f2621aa1249b304666fcde95b --- doc/source/vendor-support.rst | 2 +- os_client_config/vendors/catalyst.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 3b4a37078..34cfbe937 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -48,7 +48,7 @@ https://api.cloud.catalyst.net.nz:5000/v2.0 Region Name Human Name ============== ================ nz-por-1 Porirua, NZ -nz_wlg_1 Wellington, NZ +nz_wlg_2 Wellington, NZ ============== ================ * Image API Version is 1 diff --git a/os_client_config/vendors/catalyst.yaml b/os_client_config/vendors/catalyst.yaml index 14cdf254f..d6251930d 100644 --- a/os_client_config/vendors/catalyst.yaml +++ b/os_client_config/vendors/catalyst.yaml @@ -4,6 +4,6 @@ profile: auth_url: https://api.cloud.catalyst.net.nz:5000/v2.0 regions: - nz-por-1 - - nz_wlg_1 + - nz_wlg_2 image_api_version: '1' image_format: raw From 5f2b64bd313b2d6f5578d824537836c45763e7a5 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 2 Nov 2015 21:44:06 -0500 Subject: [PATCH 173/365] Copy values in backwards_interface differently There is the world's weirdest race condition in the unit test for this on python 3.4. Change-Id: Idee8320f03651964600ba0822edfff04f880ea4a --- os_client_config/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 483b214c8..92eb15d64 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -406,11 +406,14 @@ class OpenStackConfig(object): return cloud def _fix_backwards_interface(self, cloud): + new_cloud = {} for key in cloud.keys(): if key.endswith('endpoint_type'): target_key = key.replace('endpoint_type', 'interface') - cloud[target_key] = cloud.pop(key) - return cloud + else: + target_key = key + new_cloud[target_key] = cloud[key] + return new_cloud def get_all_clouds(self): From 48c0668b155232137cbdcf6537caca83cb870630 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 2 Nov 2015 12:35:11 -0500 Subject: [PATCH 174/365] Update auro to indicate move to neutron Auro runs neutron now. Update the config and the docs to reflect that. Change-Id: I4d2fd12b6f3d93f9e1d201b687145f1fe022e593 --- doc/source/vendor-support.rst | 4 +--- os_client_config/vendors/auro.yaml | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 34cfbe937..c3d4b3eed 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -35,9 +35,7 @@ Region Name Human Name van1 Vancouver, BC ============== ================ -* Public IPv4 is provided via NAT with Nova Floating IP -* Floating IPs are provided by Nova -* Security groups are provided by Nova +* Public IPv4 is provided via NAT with Neutron Floating IP catalyst -------- diff --git a/os_client_config/vendors/auro.yaml b/os_client_config/vendors/auro.yaml index ad721e62d..094ab7479 100644 --- a/os_client_config/vendors/auro.yaml +++ b/os_client_config/vendors/auro.yaml @@ -3,5 +3,3 @@ profile: auth: auth_url: https://api.van1.auro.io:5000/v2.0 region_name: van1 - secgroup_source: nova - floating_ip_source: nova From 796bfad22dae30e39f678c38fc9fba4f7f96032b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 6 Jun 2015 09:40:03 -0400 Subject: [PATCH 175/365] Use json for in-tree cloud data In preparation for sharing the default and vendor data with other projects, potentially even non-python ones, move the data into json format, which is slighly less exciting to read, but has more widespread standard library support. The user-facing config file will still be in yaml format, because that's easier on the eyes and it's expected to be read and edited by humans. Continue to accept yaml everywhere, because an end user may have dropped a yaml config file into a dir somewhere, and that's fine. Change-Id: I269d31e61da433ac20abb39acdde0f9f9fe12837 --- os_client_config/config.py | 17 +- os_client_config/defaults.json | 19 ++ os_client_config/defaults.py | 11 +- os_client_config/defaults.yaml | 17 -- os_client_config/schema.json | 121 ++++++++++ os_client_config/tests/test_json.py | 62 ++++++ os_client_config/vendor-schema.json | 206 ++++++++++++++++++ os_client_config/vendors/__init__.py | 7 +- os_client_config/vendors/auro.json | 9 + os_client_config/vendors/auro.yaml | 5 - os_client_config/vendors/bluebox.json | 6 + os_client_config/vendors/catalyst.json | 14 ++ os_client_config/vendors/catalyst.yaml | 9 - os_client_config/vendors/citycloud.json | 14 ++ os_client_config/vendors/citycloud.yaml | 9 - os_client_config/vendors/conoha.json | 13 ++ os_client_config/vendors/conoha.yaml | 8 - os_client_config/vendors/datacentred.json | 10 + os_client_config/vendors/datacentred.yaml | 6 - os_client_config/vendors/dreamhost.json | 10 + os_client_config/vendors/dreamhost.yaml | 6 - os_client_config/vendors/elastx.json | 9 + os_client_config/vendors/elastx.yaml | 5 - os_client_config/vendors/entercloudsuite.json | 13 ++ os_client_config/vendors/entercloudsuite.yaml | 8 - os_client_config/vendors/hp.json | 14 ++ os_client_config/vendors/hp.yaml | 9 - os_client_config/vendors/internap.json | 15 ++ os_client_config/vendors/internap.yaml | 10 - os_client_config/vendors/ovh.json | 14 ++ os_client_config/vendors/ovh.yaml | 9 - os_client_config/vendors/rackspace.json | 27 +++ os_client_config/vendors/rackspace.yaml | 21 -- os_client_config/vendors/runabove.json | 14 ++ os_client_config/vendors/runabove.yaml | 9 - os_client_config/vendors/switchengines.json | 14 ++ os_client_config/vendors/switchengines.yaml | 9 - os_client_config/vendors/ultimum.json | 9 + os_client_config/vendors/ultimum.yaml | 5 - os_client_config/vendors/unitedstack.json | 15 ++ os_client_config/vendors/unitedstack.yaml | 10 - os_client_config/vendors/vexxhost.json | 10 + os_client_config/vendors/vexxhost.yaml | 6 - test-requirements.txt | 1 + 44 files changed, 661 insertions(+), 174 deletions(-) create mode 100644 os_client_config/defaults.json delete mode 100644 os_client_config/defaults.yaml create mode 100644 os_client_config/schema.json create mode 100644 os_client_config/tests/test_json.py create mode 100644 os_client_config/vendor-schema.json create mode 100644 os_client_config/vendors/auro.json delete mode 100644 os_client_config/vendors/auro.yaml create mode 100644 os_client_config/vendors/bluebox.json create mode 100644 os_client_config/vendors/catalyst.json delete mode 100644 os_client_config/vendors/catalyst.yaml create mode 100644 os_client_config/vendors/citycloud.json delete mode 100644 os_client_config/vendors/citycloud.yaml create mode 100644 os_client_config/vendors/conoha.json delete mode 100644 os_client_config/vendors/conoha.yaml create mode 100644 os_client_config/vendors/datacentred.json delete mode 100644 os_client_config/vendors/datacentred.yaml create mode 100644 os_client_config/vendors/dreamhost.json delete mode 100644 os_client_config/vendors/dreamhost.yaml create mode 100644 os_client_config/vendors/elastx.json delete mode 100644 os_client_config/vendors/elastx.yaml create mode 100644 os_client_config/vendors/entercloudsuite.json delete mode 100644 os_client_config/vendors/entercloudsuite.yaml create mode 100644 os_client_config/vendors/hp.json delete mode 100644 os_client_config/vendors/hp.yaml create mode 100644 os_client_config/vendors/internap.json delete mode 100644 os_client_config/vendors/internap.yaml create mode 100644 os_client_config/vendors/ovh.json delete mode 100644 os_client_config/vendors/ovh.yaml create mode 100644 os_client_config/vendors/rackspace.json delete mode 100644 os_client_config/vendors/rackspace.yaml create mode 100644 os_client_config/vendors/runabove.json delete mode 100644 os_client_config/vendors/runabove.yaml create mode 100644 os_client_config/vendors/switchengines.json delete mode 100644 os_client_config/vendors/switchengines.yaml create mode 100644 os_client_config/vendors/ultimum.json delete mode 100644 os_client_config/vendors/ultimum.yaml create mode 100644 os_client_config/vendors/unitedstack.json delete mode 100644 os_client_config/vendors/unitedstack.yaml create mode 100644 os_client_config/vendors/vexxhost.json delete mode 100644 os_client_config/vendors/vexxhost.yaml diff --git a/os_client_config/config.py b/os_client_config/config.py index 92eb15d64..af9dc3b48 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -13,6 +13,7 @@ # under the License. +import json import os import warnings @@ -44,15 +45,16 @@ CONFIG_SEARCH_PATH = [ SITE_CONFIG_HOME, UNIX_SITE_CONFIG_HOME ] YAML_SUFFIXES = ('.yaml', '.yml') +JSON_SUFFIXES = ('.json',) CONFIG_FILES = [ os.path.join(d, 'clouds' + s) for d in CONFIG_SEARCH_PATH - for s in YAML_SUFFIXES + for s in YAML_SUFFIXES + JSON_SUFFIXES ] VENDOR_FILES = [ os.path.join(d, 'clouds-public' + s) for d in CONFIG_SEARCH_PATH - for s in YAML_SUFFIXES + for s in YAML_SUFFIXES + JSON_SUFFIXES ] BOOL_KEYS = ('insecure', 'cache') @@ -212,16 +214,19 @@ class OpenStackConfig(object): 'expiration', self._cache_expiration) def _load_config_file(self): - return self._load_yaml_file(self._config_files) + return self._load_yaml_json_file(self._config_files) def _load_vendor_file(self): - return self._load_yaml_file(self._vendor_files) + return self._load_yaml_json_file(self._vendor_files) - def _load_yaml_file(self, filelist): + def _load_yaml_json_file(self, filelist): for path in filelist: if os.path.exists(path): with open(path, 'r') as f: - return path, yaml.safe_load(f) + if path.endswith('json'): + return path, json.load(f) + else: + return path, yaml.safe_load(f) return (None, None) def _normalize_keys(self, config): diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json new file mode 100644 index 000000000..9239d0fe8 --- /dev/null +++ b/os_client_config/defaults.json @@ -0,0 +1,19 @@ +{ + "auth_type": "password", + "baremetal_api_version": "1", + "compute_api_version": "2", + "database_api_version": "1.0", + "disable_vendor_agent": {}, + "dns_api_version": "2", + "interface": "public", + "floating_ip_source": "neutron", + "identity_api_version": "2.0", + "image_api_use_tasks": false, + "image_api_version": "2", + "image_format": "qcow2", + "network_api_version": "2", + "object_api_version": "1", + "orchestration_api_version": "1", + "secgroup_source": "neutron", + "volume_api_version": "1" +} diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index 897cff568..c10358a10 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -12,12 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +import json import os -import yaml - -_yaml_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), 'defaults.yaml') +_json_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'defaults.json') _defaults = None @@ -34,8 +33,8 @@ def get_defaults(): cert=None, key=None, ) - with open(_yaml_path, 'r') as yaml_file: - updates = yaml.load(yaml_file.read()) + with open(_json_path, 'r') as json_file: + updates = json.load(json_file) if updates is not None: _defaults.update(updates) diff --git a/os_client_config/defaults.yaml b/os_client_config/defaults.yaml deleted file mode 100644 index ddb975cf0..000000000 --- a/os_client_config/defaults.yaml +++ /dev/null @@ -1,17 +0,0 @@ -auth_type: password -baremetal_api_version: '1' -compute_api_version: '2' -database_api_version: '1.0' -disable_vendor_agent: {} -dns_api_version: '2' -interface: public -floating_ip_source: neutron -identity_api_version: '2.0' -image_api_use_tasks: false -image_api_version: '2' -image_format: qcow2 -network_api_version: '2' -object_api_version: '1' -orchestration_api_version: '1' -secgroup_source: neutron -volume_api_version: '1' diff --git a/os_client_config/schema.json b/os_client_config/schema.json new file mode 100644 index 000000000..dfd1f4a9b --- /dev/null +++ b/os_client_config/schema.json @@ -0,0 +1,121 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "https://git.openstack.org/cgit/openstack/cloud-data/plain/schema.json#", + "type": "object", + "properties": { + "auth_type": { + "name": "Auth Type", + "description": "Name of authentication plugin to be used", + "default": "password", + "type": "string" + }, + "disable_vendor_agent": { + "name": "Disable Vendor Agent Properties", + "description": "Image properties required to disable vendor agent", + "type": "object", + "properties": {} + }, + "floating_ip_source": { + "name": "Floating IP Source", + "description": "Which service provides Floating IPs", + "enum": [ "neutron", "nova", "None" ], + "default": "neutron" + }, + "image_api_use_tasks": { + "name": "Image Task API", + "description": "Does the cloud require the Image Task API", + "default": false, + "type": "boolean" + }, + "image_format": { + "name": "Image Format", + "description": "Format for uploaded Images", + "default": "qcow2", + "type": "string" + }, + "interface": { + "name": "API Interface", + "description": "Which API Interface should connections hit", + "default": "public", + "enum": [ "public", "internal", "admin" ] + }, + "secgroup_source": { + "name": "Security Group Source", + "description": "Which service provides security groups", + "default": "neutron", + "enum": [ "neutron", "nova", "None" ] + }, + "baremetal_api_version": { + "name": "Baremetal API Service Type", + "description": "Baremetal API Service Type", + "default": "1", + "type": "string" + }, + "compute_api_version": { + "name": "Compute API Version", + "description": "Compute API Version", + "default": "2", + "type": "string" + }, + "database_api_version": { + "name": "Database API Version", + "description": "Database API Version", + "default": "1.0", + "type": "string" + }, + "dns_api_version": { + "name": "DNS API Version", + "description": "DNS API Version", + "default": "2", + "type": "string" + }, + "identity_api_version": { + "name": "Identity API Version", + "description": "Identity API Version", + "default": "2", + "type": "string" + }, + "image_api_version": { + "name": "Image API Version", + "description": "Image API Version", + "default": "1", + "type": "string" + }, + "network_api_version": { + "name": "Network API Version", + "description": "Network API Version", + "default": "2", + "type": "string" + }, + "object_api_version": { + "name": "Object Storage API Version", + "description": "Object Storage API Version", + "default": "1", + "type": "string" + }, + "volume_api_version": { + "name": "Volume API Version", + "description": "Volume API Version", + "default": "2", + "type": "string" + } + }, + "required": [ + "auth_type", + "baremetal_api_version", + "compute_api_version", + "database_api_version", + "disable_vendor_agent", + "dns_api_version", + "floating_ip_source", + "identity_api_version", + "image_api_use_tasks", + "image_api_version", + "image_format", + "interface", + "network_api_version", + "object_api_version", + "secgroup_source", + "volume_api_version" + ] +} diff --git a/os_client_config/tests/test_json.py b/os_client_config/tests/test_json.py new file mode 100644 index 000000000..f618f3b29 --- /dev/null +++ b/os_client_config/tests/test_json.py @@ -0,0 +1,62 @@ +# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import glob +import json +import os + +import jsonschema +from testtools import content + +from os_client_config import defaults +from os_client_config.tests import base + + +class TestConfig(base.TestCase): + + def json_diagnostics(self, exc_info): + self.addDetail('filename', content.text_content(self.filename)) + for error in sorted(self.validator.iter_errors(self.json_data)): + self.addDetail('jsonschema', content.text_content(str(error))) + + def test_defaults_valid_json(self): + _schema_path = os.path.join( + os.path.dirname(os.path.realpath(defaults.__file__)), + 'schema.json') + schema = json.load(open(_schema_path, 'r')) + self.validator = jsonschema.Draft4Validator(schema) + self.addOnException(self.json_diagnostics) + + self.filename = os.path.join( + os.path.dirname(os.path.realpath(defaults.__file__)), + 'defaults.json') + self.json_data = json.load(open(self.filename, 'r')) + + self.assertTrue(self.validator.is_valid(self.json_data)) + + def test_vendors_valid_json(self): + _schema_path = os.path.join( + os.path.dirname(os.path.realpath(defaults.__file__)), + 'vendor-schema.json') + schema = json.load(open(_schema_path, 'r')) + self.validator = jsonschema.Draft4Validator(schema) + self.addOnException(self.json_diagnostics) + + _vendors_path = os.path.join( + os.path.dirname(os.path.realpath(defaults.__file__)), + 'vendors') + for self.filename in glob.glob(os.path.join(_vendors_path, '*.json')): + self.json_data = json.load(open(self.filename, 'r')) + + self.assertTrue(self.validator.is_valid(self.json_data)) diff --git a/os_client_config/vendor-schema.json b/os_client_config/vendor-schema.json new file mode 100644 index 000000000..e3fb57359 --- /dev/null +++ b/os_client_config/vendor-schema.json @@ -0,0 +1,206 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "https://git.openstack.org/cgit/openstack/cloud-data/plain/vendor-schema.json#", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "profile": { + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "auth_url": { + "name": "Auth URL", + "description": "URL of the primary Keystone endpoint", + "type": "string" + } + } + }, + "auth_type": { + "name": "Auth Type", + "description": "Name of authentication plugin to be used", + "default": "password", + "type": "string" + }, + "disable_vendor_agent": { + "name": "Disable Vendor Agent Properties", + "description": "Image properties required to disable vendor agent", + "type": "object", + "properties": {} + }, + "floating_ip_source": { + "name": "Floating IP Source", + "description": "Which service provides Floating IPs", + "enum": [ "neutron", "nova", "None" ], + "default": "neutron" + }, + "image_api_use_tasks": { + "name": "Image Task API", + "description": "Does the cloud require the Image Task API", + "default": false, + "type": "boolean" + }, + "image_format": { + "name": "Image Format", + "description": "Format for uploaded Images", + "default": "qcow2", + "type": "string" + }, + "interface": { + "name": "API Interface", + "description": "Which API Interface should connections hit", + "default": "public", + "enum": [ "public", "internal", "admin" ] + }, + "secgroup_source": { + "name": "Security Group Source", + "description": "Which service provides security groups", + "enum": [ "neutron", "nova", "None" ], + "default": "neutron" + }, + "compute_api_service_name": { + "name": "Compute API Service Name", + "description": "Compute API Service Name", + "type": "string" + }, + "database_api_service_name": { + "name": "Database API Service Name", + "description": "Database API Service Name", + "type": "string" + }, + "dns_api_service_name": { + "name": "DNS API Service Name", + "description": "DNS API Service Name", + "type": "string" + }, + "identity_api_service_name": { + "name": "Identity API Service Name", + "description": "Identity API Service Name", + "type": "string" + }, + "image_api_service_name": { + "name": "Image API Service Name", + "description": "Image API Service Name", + "type": "string" + }, + "volume_api_service_name": { + "name": "Volume API Service Name", + "description": "Volume API Service Name", + "type": "string" + }, + "network_api_service_name": { + "name": "Network API Service Name", + "description": "Network API Service Name", + "type": "string" + }, + "object_api_service_name": { + "name": "Object Storage API Service Name", + "description": "Object Storage API Service Name", + "type": "string" + }, + "baremetal_api_service_name": { + "name": "Baremetal API Service Name", + "description": "Baremetal API Service Name", + "type": "string" + }, + "compute_api_service_type": { + "name": "Compute API Service Type", + "description": "Compute API Service Type", + "type": "string" + }, + "database_api_service_type": { + "name": "Database API Service Type", + "description": "Database API Service Type", + "type": "string" + }, + "dns_api_service_type": { + "name": "DNS API Service Type", + "description": "DNS API Service Type", + "type": "string" + }, + "identity_api_service_type": { + "name": "Identity API Service Type", + "description": "Identity API Service Type", + "type": "string" + }, + "image_api_service_type": { + "name": "Image API Service Type", + "description": "Image API Service Type", + "type": "string" + }, + "volume_api_service_type": { + "name": "Volume API Service Type", + "description": "Volume API Service Type", + "type": "string" + }, + "network_api_service_type": { + "name": "Network API Service Type", + "description": "Network API Service Type", + "type": "string" + }, + "object_api_service_type": { + "name": "Object Storage API Service Type", + "description": "Object Storage API Service Type", + "type": "string" + }, + "baremetal_api_version": { + "name": "Baremetal API Service Type", + "description": "Baremetal API Service Type", + "type": "string" + }, + "compute_api_version": { + "name": "Compute API Version", + "description": "Compute API Version", + "type": "string" + }, + "database_api_version": { + "name": "Database API Version", + "description": "Database API Version", + "type": "string" + }, + "dns_api_version": { + "name": "DNS API Version", + "description": "DNS API Version", + "type": "string" + }, + "identity_api_version": { + "name": "Identity API Version", + "description": "Identity API Version", + "type": "string" + }, + "image_api_version": { + "name": "Image API Version", + "description": "Image API Version", + "type": "string" + }, + "volume_api_version": { + "name": "Volume API Version", + "description": "Volume API Version", + "type": "string" + }, + "network_api_version": { + "name": "Network API Version", + "description": "Network API Version", + "type": "string" + }, + "object_api_version": { + "name": "Object Storage API Version", + "description": "Object Storage API Version", + "type": "string" + }, + "baremetal_api_version": { + "name": "Baremetal API Version", + "description": "Baremetal API Version", + "type": "string" + } + } + } + }, + "required": [ + "name", + "profile" + ] +} diff --git a/os_client_config/vendors/__init__.py b/os_client_config/vendors/__init__.py index 367e3182b..3e1d20a5a 100644 --- a/os_client_config/vendors/__init__.py +++ b/os_client_config/vendors/__init__.py @@ -13,6 +13,7 @@ # under the License. import glob +import json import os import yaml @@ -27,6 +28,10 @@ def get_profile(profile_name): _vendor_defaults = {} for vendor in glob.glob(os.path.join(_vendors_path, '*.yaml')): with open(vendor, 'r') as f: - vendor_data = yaml.load(f) + vendor_data = yaml.safe_load(f) + _vendor_defaults[vendor_data['name']] = vendor_data['profile'] + for vendor in glob.glob(os.path.join(_vendors_path, '*.json')): + with open(vendor, 'r') as f: + vendor_data = json.load(f) _vendor_defaults[vendor_data['name']] = vendor_data['profile'] return _vendor_defaults.get(profile_name) diff --git a/os_client_config/vendors/auro.json b/os_client_config/vendors/auro.json new file mode 100644 index 000000000..1e59f01d4 --- /dev/null +++ b/os_client_config/vendors/auro.json @@ -0,0 +1,9 @@ +{ + "name": "auro", + "profile": { + "auth": { + "auth_url": "https://api.van1.auro.io:5000/v2.0" + }, + "region_name": "van1" + } +} diff --git a/os_client_config/vendors/auro.yaml b/os_client_config/vendors/auro.yaml deleted file mode 100644 index 094ab7479..000000000 --- a/os_client_config/vendors/auro.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: auro -profile: - auth: - auth_url: https://api.van1.auro.io:5000/v2.0 - region_name: van1 diff --git a/os_client_config/vendors/bluebox.json b/os_client_config/vendors/bluebox.json new file mode 100644 index 000000000..2227aac81 --- /dev/null +++ b/os_client_config/vendors/bluebox.json @@ -0,0 +1,6 @@ +{ + "name": "bluebox", + "profile": { + "region_name": "RegionOne" + } +} diff --git a/os_client_config/vendors/catalyst.json b/os_client_config/vendors/catalyst.json new file mode 100644 index 000000000..ddde83825 --- /dev/null +++ b/os_client_config/vendors/catalyst.json @@ -0,0 +1,14 @@ +{ + "name": "catalyst", + "profile": { + "auth": { + "auth_url": "https://api.cloud.catalyst.net.nz:5000/v2.0" + }, + "regions": [ + "nz-por-1", + "nz_wlg_2" + ], + "image_api_version": "1", + "image_format": "raw" + } +} diff --git a/os_client_config/vendors/catalyst.yaml b/os_client_config/vendors/catalyst.yaml deleted file mode 100644 index d6251930d..000000000 --- a/os_client_config/vendors/catalyst.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: catalyst -profile: - auth: - auth_url: https://api.cloud.catalyst.net.nz:5000/v2.0 - regions: - - nz-por-1 - - nz_wlg_2 - image_api_version: '1' - image_format: raw diff --git a/os_client_config/vendors/citycloud.json b/os_client_config/vendors/citycloud.json new file mode 100644 index 000000000..f6c57c7b5 --- /dev/null +++ b/os_client_config/vendors/citycloud.json @@ -0,0 +1,14 @@ +{ + "name": "citycloud", + "profile": { + "auth": { + "auth_url": "https://identity1.citycloud.com:5000/v3/" + }, + "regions": [ + "Lon1", + "Sto2", + "Kna1" + ], + "identity_api_version": "3" + } +} diff --git a/os_client_config/vendors/citycloud.yaml b/os_client_config/vendors/citycloud.yaml deleted file mode 100644 index 9ee0b9b4f..000000000 --- a/os_client_config/vendors/citycloud.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: citycloud -profile: - auth: - auth_url: https://identity1.citycloud.com:5000/v3/ - regions: - - Lon1 - - Sto2 - - Kna1 - identity_api_version: '3' diff --git a/os_client_config/vendors/conoha.json b/os_client_config/vendors/conoha.json new file mode 100644 index 000000000..28f1b2754 --- /dev/null +++ b/os_client_config/vendors/conoha.json @@ -0,0 +1,13 @@ +{ + "name": "conoha", + "profile": { + "auth": { + "auth_url": "https://identity.{region_name}.conoha.io/v2.0" + }, + "regions": [ + "sin1", + "lon1", + "tyo1" + ] + } +} diff --git a/os_client_config/vendors/conoha.yaml b/os_client_config/vendors/conoha.yaml deleted file mode 100644 index 1ed4063ec..000000000 --- a/os_client_config/vendors/conoha.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: conoha -profile: - auth: - auth_url: https://identity.{region_name}.conoha.io/v2.0 - regions: - - sin1 - - lon1 - - tyo1 diff --git a/os_client_config/vendors/datacentred.json b/os_client_config/vendors/datacentred.json new file mode 100644 index 000000000..1fb4dbb09 --- /dev/null +++ b/os_client_config/vendors/datacentred.json @@ -0,0 +1,10 @@ +{ + "name": "datacentred", + "profile": { + "auth": { + "auth_url": "https://compute.datacentred.io:5000/v2.0" + }, + "region-name": "sal01", + "image_api_version": "1" + } +} diff --git a/os_client_config/vendors/datacentred.yaml b/os_client_config/vendors/datacentred.yaml deleted file mode 100644 index 5c0a5ed1a..000000000 --- a/os_client_config/vendors/datacentred.yaml +++ /dev/null @@ -1,6 +0,0 @@ -name: datacentred -profile: - auth: - auth_url: https://compute.datacentred.io:5000/v2.0 - region-name: sal01 - image_api_version: '1' diff --git a/os_client_config/vendors/dreamhost.json b/os_client_config/vendors/dreamhost.json new file mode 100644 index 000000000..8580826e1 --- /dev/null +++ b/os_client_config/vendors/dreamhost.json @@ -0,0 +1,10 @@ +{ + "name": "dreamhost", + "profile": { + "auth": { + "auth_url": "https://keystone.dream.io/v2.0" + }, + "region_name": "RegionOne", + "image_format": "raw" + } +} diff --git a/os_client_config/vendors/dreamhost.yaml b/os_client_config/vendors/dreamhost.yaml deleted file mode 100644 index 3cd395a17..000000000 --- a/os_client_config/vendors/dreamhost.yaml +++ /dev/null @@ -1,6 +0,0 @@ -name: dreamhost -profile: - auth: - auth_url: https://keystone.dream.io/v2.0 - region_name: RegionOne - image_format: raw diff --git a/os_client_config/vendors/elastx.json b/os_client_config/vendors/elastx.json new file mode 100644 index 000000000..cac755e8f --- /dev/null +++ b/os_client_config/vendors/elastx.json @@ -0,0 +1,9 @@ +{ + "name": "elastx", + "profile": { + "auth": { + "auth_url": "https://ops.elastx.net:5000/v2.0" + }, + "region_name": "regionOne" + } +} diff --git a/os_client_config/vendors/elastx.yaml b/os_client_config/vendors/elastx.yaml deleted file mode 100644 index 810e12ede..000000000 --- a/os_client_config/vendors/elastx.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: elastx -profile: - auth: - auth_url: https://ops.elastx.net:5000/v2.0 - region_name: regionOne diff --git a/os_client_config/vendors/entercloudsuite.json b/os_client_config/vendors/entercloudsuite.json new file mode 100644 index 000000000..826c25f7f --- /dev/null +++ b/os_client_config/vendors/entercloudsuite.json @@ -0,0 +1,13 @@ +{ + "name": "entercloudsuite", + "profile": { + "auth": { + "auth_url": "https://api.entercloudsuite.com/v2.0" + }, + "regions": [ + "it-mil1", + "nl-ams1", + "de-fra1" + ] + } +} diff --git a/os_client_config/vendors/entercloudsuite.yaml b/os_client_config/vendors/entercloudsuite.yaml deleted file mode 100644 index f68bcf674..000000000 --- a/os_client_config/vendors/entercloudsuite.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: entercloudsuite -profile: - auth: - auth_url: https://api.entercloudsuite.com/v2.0 - regions: - - it-mil1 - - nl-ams1 - - de-fra1 diff --git a/os_client_config/vendors/hp.json b/os_client_config/vendors/hp.json new file mode 100644 index 000000000..10789a91b --- /dev/null +++ b/os_client_config/vendors/hp.json @@ -0,0 +1,14 @@ +{ + "name": "hp", + "profile": { + "auth": { + "auth_url": "https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0" + }, + "regions": [ + "region-a.geo-1", + "region-b.geo-1" + ], + "dns_service_type": "hpext:dns", + "image_api_version": "1" + } +} diff --git a/os_client_config/vendors/hp.yaml b/os_client_config/vendors/hp.yaml deleted file mode 100644 index a0544df82..000000000 --- a/os_client_config/vendors/hp.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: hp -profile: - auth: - auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 - regions: - - region-a.geo-1 - - region-b.geo-1 - dns_service_type: hpext:dns - image_api_version: '1' diff --git a/os_client_config/vendors/internap.json b/os_client_config/vendors/internap.json new file mode 100644 index 000000000..9b27536d5 --- /dev/null +++ b/os_client_config/vendors/internap.json @@ -0,0 +1,15 @@ +{ + "name": "internap", + "profile": { + "auth": { + "auth_url": "https://identity.api.cloud.iweb.com/v2.0" + }, + "regions": [ + "ams01", + "da01", + "nyj01" + ], + "image_api_version": "1", + "floating_ip_source": "None" + } +} diff --git a/os_client_config/vendors/internap.yaml b/os_client_config/vendors/internap.yaml deleted file mode 100644 index 48cd960eb..000000000 --- a/os_client_config/vendors/internap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: internap -profile: - auth: - auth_url: https://identity.api.cloud.iweb.com/v2.0 - regions: - - ams01 - - da01 - - nyj01 - image_api_version: '1' - floating_ip_source: None diff --git a/os_client_config/vendors/ovh.json b/os_client_config/vendors/ovh.json new file mode 100644 index 000000000..cfd234b64 --- /dev/null +++ b/os_client_config/vendors/ovh.json @@ -0,0 +1,14 @@ +{ + "name": "ovh", + "profile": { + "auth": { + "auth_url": "https://auth.cloud.ovh.net/v2.0" + }, + "regions": [ + "GRA1", + "SBG1" + ], + "image_format": "raw", + "floating_ip_source": "None" + } +} diff --git a/os_client_config/vendors/ovh.yaml b/os_client_config/vendors/ovh.yaml deleted file mode 100644 index 52b91a466..000000000 --- a/os_client_config/vendors/ovh.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: ovh -profile: - auth: - auth_url: https://auth.cloud.ovh.net/v2.0 - regions: - - GRA1 - - SBG1 - image_format: raw - floating_ip_source: None diff --git a/os_client_config/vendors/rackspace.json b/os_client_config/vendors/rackspace.json new file mode 100644 index 000000000..582e1225c --- /dev/null +++ b/os_client_config/vendors/rackspace.json @@ -0,0 +1,27 @@ +{ + "name": "rackspace", + "profile": { + "auth": { + "auth_url": "https://identity.api.rackspacecloud.com/v2.0/" + }, + "regions": [ + "DFW", + "HKG", + "IAD", + "ORD", + "SYD", + "LON" + ], + "database_service_type": "rax:database", + "compute_service_name": "cloudServersOpenStack", + "image_api_use_tasks": true, + "image_format": "vhd", + "floating_ip_source": "None", + "secgroup_source": "None", + "disable_vendor_agent": { + "vm_mode": "hvm", + "xenapi_use_agent": false + }, + "has_network": false + } +} diff --git a/os_client_config/vendors/rackspace.yaml b/os_client_config/vendors/rackspace.yaml deleted file mode 100644 index a28d49366..000000000 --- a/os_client_config/vendors/rackspace.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: rackspace -profile: - auth: - auth_url: https://identity.api.rackspacecloud.com/v2.0/ - regions: - - DFW - - HKG - - IAD - - ORD - - SYD - - LON - database_service_type: rax:database - compute_service_name: cloudServersOpenStack - image_api_use_tasks: true - image_format: vhd - floating_ip_source: None - secgroup_source: None - disable_vendor_agent: - vm_mode: hvm - xenapi_use_agent: false - has_network: false diff --git a/os_client_config/vendors/runabove.json b/os_client_config/vendors/runabove.json new file mode 100644 index 000000000..56dd9453c --- /dev/null +++ b/os_client_config/vendors/runabove.json @@ -0,0 +1,14 @@ +{ + "name": "runabove", + "profile": { + "auth": { + "auth_url": "https://auth.runabove.io/v2.0" + }, + "regions": [ + "BHS-1", + "SBG-1" + ], + "image_format": "qcow2", + "floating_ip_source": "None" + } +} diff --git a/os_client_config/vendors/runabove.yaml b/os_client_config/vendors/runabove.yaml deleted file mode 100644 index 34528941a..000000000 --- a/os_client_config/vendors/runabove.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: runabove -profile: - auth: - auth_url: https://auth.runabove.io/v2.0 - regions: - - BHS-1 - - SBG-1 - image_format: qcow2 - floating_ip_source: None diff --git a/os_client_config/vendors/switchengines.json b/os_client_config/vendors/switchengines.json new file mode 100644 index 000000000..8a7c566b8 --- /dev/null +++ b/os_client_config/vendors/switchengines.json @@ -0,0 +1,14 @@ +{ + "name": "switchengines", + "profile": { + "auth": { + "auth_url": "https://keystone.cloud.switch.ch:5000/v2.0" + }, + "regions": [ + "LS", + "ZH" + ], + "image_api_use_tasks": true, + "image_format": "raw" + } +} diff --git a/os_client_config/vendors/switchengines.yaml b/os_client_config/vendors/switchengines.yaml deleted file mode 100644 index ff6c50517..000000000 --- a/os_client_config/vendors/switchengines.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: switchengines -profile: - auth: - auth_url: https://keystone.cloud.switch.ch:5000/v2.0 - regions: - - LS - - ZH - image_api_use_tasks: true - image_format: raw diff --git a/os_client_config/vendors/ultimum.json b/os_client_config/vendors/ultimum.json new file mode 100644 index 000000000..ada6e3de7 --- /dev/null +++ b/os_client_config/vendors/ultimum.json @@ -0,0 +1,9 @@ +{ + "name": "ultimum", + "profile": { + "auth": { + "auth_url": "https://console.ultimum-cloud.com:5000/v2.0" + }, + "region-name": "RegionOne" + } +} diff --git a/os_client_config/vendors/ultimum.yaml b/os_client_config/vendors/ultimum.yaml deleted file mode 100644 index 866117491..000000000 --- a/os_client_config/vendors/ultimum.yaml +++ /dev/null @@ -1,5 +0,0 @@ -name: ultimum -profile: - auth: - auth_url: https://console.ultimum-cloud.com:5000/v2.0 - region-name: RegionOne diff --git a/os_client_config/vendors/unitedstack.json b/os_client_config/vendors/unitedstack.json new file mode 100644 index 000000000..41f45851a --- /dev/null +++ b/os_client_config/vendors/unitedstack.json @@ -0,0 +1,15 @@ +{ + "name": "unitedstack", + "profile": { + "auth": { + "auth_url": "https://identity.api.ustack.com/v3" + }, + "regions": [ + "bj1", + "gd1" + ], + "identity_api_version": "3", + "image_format": "raw", + "floating_ip_source": "None" + } +} diff --git a/os_client_config/vendors/unitedstack.yaml b/os_client_config/vendors/unitedstack.yaml deleted file mode 100644 index c6d5cc20e..000000000 --- a/os_client_config/vendors/unitedstack.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: unitedstack -profile: - auth: - auth_url: https://identity.api.ustack.com/v3 - regions: - - bj1 - - gd1 - identity_api_version: '3' - image_format: raw - floating_ip_source: None diff --git a/os_client_config/vendors/vexxhost.json b/os_client_config/vendors/vexxhost.json new file mode 100644 index 000000000..25911cae1 --- /dev/null +++ b/os_client_config/vendors/vexxhost.json @@ -0,0 +1,10 @@ +{ + "name": "vexxhost", + "profile": { + "auth": { + "auth_url": "http://auth.api.thenebulacloud.com:5000/v2.0/" + }, + "region_name": "ca-ymq-1", + "floating_ip_source": "None" + } +} diff --git a/os_client_config/vendors/vexxhost.yaml b/os_client_config/vendors/vexxhost.yaml deleted file mode 100644 index 4a0ba271b..000000000 --- a/os_client_config/vendors/vexxhost.yaml +++ /dev/null @@ -1,6 +0,0 @@ -name: vexxhost -profile: - auth: - auth_url: http://auth.api.thenebulacloud.com:5000/v2.0/ - region_name: ca-ymq-1 - floating_ip_source: None diff --git a/test-requirements.txt b/test-requirements.txt index 62f3e8831..9a02b042c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,6 +8,7 @@ coverage>=3.6 extras fixtures>=0.3.14 discover +jsonschema>=2.0.0,<3.0.0,!=2.5.0 python-keystoneclient>=1.1.0 python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 From 27678e5d948296986c31705c8405f569d2c06553 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 3 Nov 2015 08:11:53 -0500 Subject: [PATCH 176/365] Update conoha's vendor profile to include SJC Turns out the lon region is not really a thing. Change-Id: Ib315c301d5f8b589006a61d2a255a6b295b1b9a5 --- doc/source/vendor-support.rst | 2 +- os_client_config/vendors/conoha.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index c3d4b3eed..a9dd5ef8c 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -78,7 +78,7 @@ Region Name Human Name ============== ================ tyo1 Tokyo, JP sin1 Singapore -lon1 London, UK +sjc1 San Jose, CA ============== ================ * Image upload is not supported diff --git a/os_client_config/vendors/conoha.json b/os_client_config/vendors/conoha.json index 28f1b2754..8e33ca41b 100644 --- a/os_client_config/vendors/conoha.json +++ b/os_client_config/vendors/conoha.json @@ -6,7 +6,7 @@ }, "regions": [ "sin1", - "lon1", + "sjc1", "tyo1" ] } From a494b31b85cf320c891260e837a69867348eca78 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 26 Oct 2015 10:31:15 +0900 Subject: [PATCH 177/365] Add methods for getting Session and Client objects These come originally from the shade library, but are helpful for things like the client libs themselves. Once one has a CloudConfig, there is really one and only one correct way to get both a session and a Client. Change-Id: I1b4d4321828864fddab85a127fbf63f4c8384ab9 --- os_client_config/cloud_config.py | 139 ++++++++++++++++++++ os_client_config/tests/test_cloud_config.py | 135 +++++++++++++++++++ test-requirements.txt | 2 + 3 files changed, 276 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 63ff8d29b..6085a6496 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -14,6 +14,11 @@ import warnings +from keystoneauth1 import plugin +from keystoneauth1 import session + +from os_client_config import exceptions + class CloudConfig(object): def __init__(self, name, region, config, @@ -25,6 +30,7 @@ class CloudConfig(object): self._force_ipv4 = force_ipv4 self._auth = auth_plugin self._openstack_config = openstack_config + self._keystone_session = None def __getattr__(self, key): """Return arbitrary attributes.""" @@ -119,6 +125,139 @@ class CloudConfig(object): """Return a keystoneauth plugin from the auth credentials.""" return self._auth + def get_session(self): + """Return a keystoneauth session based on the auth credentials.""" + if self._keystone_session is None: + if not self._auth: + raise exceptions.OpenStackConfigException( + "Problem with auth parameters") + (verify, cert) = self.get_requests_verify_args() + self._keystone_session = session.Session( + auth=self._auth, + verify=verify, + cert=cert, + timeout=self.config['api_timeout']) + return self._keystone_session + + def get_session_endpoint(self, service_key): + """Return the endpoint from config or the catalog. + + If a configuration lists an explicit endpoint for a service, + return that. Otherwise, fetch the service catalog from the + keystone session and return the appropriate endpoint. + + :param service_key: Generic key for service, such as 'compute' or + 'network' + + :returns: Endpoint for the service, or None if not found + """ + + override_endpoint = self.get_endpoint(service_key) + if override_endpoint: + return override_endpoint + # keystone is a special case in keystone, because what? + session = self.get_session() + if service_key == 'identity': + endpoint = session.get_endpoint(interface=plugin.AUTH_INTERFACE) + else: + endpoint = session.get_endpoint( + service_type=self.get_service_type(service_key), + service_name=self.get_service_name(service_key), + interface=self.get_interface(service_key), + region_name=self.region) + return endpoint + + def get_legacy_client( + self, service_key, client_class, interface_key=None, + pass_version_arg=True, **kwargs): + """Return a legacy OpenStack client object for the given config. + + Most of the OpenStack python-*client libraries have the same + interface for their client constructors, but there are several + parameters one wants to pass given a :class:`CloudConfig` object. + + In the future, OpenStack API consumption should be done through + the OpenStack SDK, but that's not ready yet. This is for getting + Client objects from python-*client only. + + :param service_key: Generic key for service, such as 'compute' or + 'network' + :param client_class: Class of the client to be instantiated. This + should be the unversioned version if there + is one, such as novaclient.client.Client, or + the versioned one, such as + neutronclient.v2_0.client.Client if there isn't + :param interface_key: (optional) Some clients, such as glanceclient + only accept the parameter 'interface' instead + of 'endpoint_type' - this is a get-out-of-jail + parameter for those until they can be aligned. + os-client-config understands this to be the + case if service_key is image, so this is really + only for use with other unknown broken clients. + :param pass_version_arg: (optional) If a versioned Client constructor + was passed to client_class, set this to + False, which will tell get_client to not + pass a version parameter. os-client-config + already understand that this is the + case for network, so it can be omitted in + that case. + :param kwargs: (optional) keyword args are passed through to the + Client constructor, so this is in case anything + additional needs to be passed in. + """ + # Because of course swift is different + if service_key == 'object-store': + return self._get_swift_client(client_class=client_class, **kwargs) + interface = self.get_interface(service_key) + # trigger exception on lack of service + endpoint = self.get_session_endpoint(service_key) + + if not interface_key: + if service_key == 'image': + interface_key = 'interface' + else: + interface_key = 'endpoint_type' + if service_key == 'network': + pass_version_arg = False + + constructor_args = dict( + session=self.get_session(), + service_name=self.get_service_name(service_key), + service_type=self.get_service_type(service_key), + region_name=self.region) + + if service_key == 'image': + # os-client-config does not depend on glanceclient, but if + # the user passed in glanceclient.client.Client, which they + # would need to do if they were requesting 'image' - then + # they necessarily have glanceclient installed + from glanceclient.common import utils as glance_utils + endpoint, version = glance_utils.strip_version(endpoint) + constructor_args['endpoint'] = endpoint + constructor_args.update(kwargs) + constructor_args[interface_key] = interface + if pass_version_arg: + version = self.get_api_version(service_key) + constructor_args['version'] = version + return client_class(**constructor_args) + + def _get_swift_client(self, client_class, **kwargs): + session = self.get_session() + token = session.get_token() + endpoint = self.get_session_endpoint(service_key='object-store') + if not endpoint: + return None + return client_class( + preauthurl=endpoint, + preauthtoken=token, + auth_version=self.get_api_version('identity'), + os_options=dict( + auth_token=token, + object_storage_url=endpoint, + region_name=self.get_region_name()), + timeout=self.api_timeout, + ) + def get_cache_expiration_time(self): if self._openstack_config: return self._openstack_config.get_cache_expiration_time() diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index c9317ad00..f5ceac7b7 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -12,7 +12,13 @@ import copy +from keystoneauth1 import plugin as ksa_plugin +from keystoneauth1 import session as ksa_session +import mock + from os_client_config import cloud_config +from os_client_config import defaults +from os_client_config import exceptions from os_client_config.tests import base @@ -142,3 +148,132 @@ class TestCloudConfig(base.TestCase): cc.get_endpoint('image')) self.assertEqual(None, cc.get_service_name('compute')) self.assertEqual('locks', cc.get_service_name('identity')) + + def test_get_session_no_auth(self): + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig("test1", "region-al", config_dict) + self.assertRaises( + exceptions.OpenStackConfigException, + cc.get_session) + + @mock.patch.object(ksa_session, 'Session') + def test_get_session(self, mock_session): + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_session() + mock_session.assert_called_with( + auth=mock.ANY, + verify=True, cert=None, timeout=None) + + @mock.patch.object(ksa_session, 'Session') + def test_override_session_endpoint(self, mock_session): + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + self.assertEqual( + cc.get_session_endpoint('compute'), + fake_services_dict['compute_endpoint']) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session') + def test_session_endpoint_identity(self, mock_get_session): + mock_session = mock.Mock() + mock_get_session.return_value = mock_session + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_session_endpoint('identity') + mock_session.get_endpoint.assert_called_with( + interface=ksa_plugin.AUTH_INTERFACE) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session') + def test_session_endpoint(self, mock_get_session): + mock_session = mock.Mock() + mock_get_session.return_value = mock_session + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_session_endpoint('orchestration') + mock_session.get_endpoint.assert_called_with( + interface='public', + service_name=None, + region_name='region-al', + service_type='orchestration') + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_object_store(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('object-store', mock_client) + mock_client.assert_called_with( + preauthtoken=mock.ANY, + os_options={ + 'auth_token': mock.ANY, + 'region_name': 'region-al', + 'object_storage_url': 'http://example.com/v2' + }, + preauthurl='http://example.com/v2', + auth_version='2.0', + timeout=None) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_image(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('image', mock_client) + mock_client.assert_called_with( + version='2', + service_name=None, + endpoint='http://example.com', + region_name='region-al', + interface='public', + session=mock.ANY, + # Not a typo - the config dict above overrides this + service_type='mage' + ) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_network(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('network', mock_client) + mock_client.assert_called_with( + endpoint_type='public', + region_name='region-al', + service_type='network', + session=mock.ANY, + service_name=None) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_compute(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('compute', mock_client) + mock_client.assert_called_with( + version=2, + endpoint_type='public', + region_name='region-al', + service_type='compute', + session=mock.ANY, + service_name=None) diff --git a/test-requirements.txt b/test-requirements.txt index 9a02b042c..70530517e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,6 +9,8 @@ extras fixtures>=0.3.14 discover jsonschema>=2.0.0,<3.0.0,!=2.5.0 +mock>=1.2 +python-glanceclient>=0.18.0 python-keystoneclient>=1.1.0 python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 From 335ed4a6944ad32759ad810aa6a4da599bc358fd Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 3 Nov 2015 10:45:20 -0500 Subject: [PATCH 178/365] Add logging module support The _log.py module is from shade and is just basic logging support that does not emit warnings if the consumer does not have logging enabled. Change-Id: Id4639763cf488eb7cd0c27904be055a7843e287f --- os_client_config/_log.py | 28 ++++++++++++++++++++++++++++ os_client_config/cloud_config.py | 2 ++ 2 files changed, 30 insertions(+) create mode 100644 os_client_config/_log.py diff --git a/os_client_config/_log.py b/os_client_config/_log.py new file mode 100644 index 000000000..ff2f2eac7 --- /dev/null +++ b/os_client_config/_log.py @@ -0,0 +1,28 @@ +# Copyright (c) 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + + +class NullHandler(logging.Handler): + def emit(self, record): + pass + + +def setup_logging(name): + log = logging.getLogger(name) + if len(log.handlers) == 0: + h = NullHandler() + log.addHandler(h) + return log diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 6085a6496..fe5c3021d 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -17,6 +17,7 @@ import warnings from keystoneauth1 import plugin from keystoneauth1 import session +from os_client_config import _log from os_client_config import exceptions @@ -27,6 +28,7 @@ class CloudConfig(object): self.name = name self.region = region self.config = config + self.log = _log.setup_logging(__name__) self._force_ipv4 = force_ipv4 self._auth = auth_plugin self._openstack_config = openstack_config From 6d957b01e55637ad4e8ce69c31cf3b3f64881c57 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 3 Nov 2015 10:42:53 -0500 Subject: [PATCH 179/365] Disable spurious urllib warnings Since we can return a keystoneauth1 Session correctly, disable the stupid warnings that are evil and never should be emitted. Change-Id: I128e4587ab07a20c7c5da745b12e4f5d0a3ee5a7 --- os_client_config/cloud_config.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index fe5c3021d..17941c970 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -20,6 +20,15 @@ from keystoneauth1 import session from os_client_config import _log from os_client_config import exceptions +# Importing these for later but not disabling for now +try: + from requests.packages.urllib3 import exceptions as urllib_exc +except ImportError: + try: + from urllib3 import exceptions as urllib_exc + except ImportError: + urllib_exc = None + class CloudConfig(object): def __init__(self, name, region, config, @@ -134,6 +143,16 @@ class CloudConfig(object): raise exceptions.OpenStackConfigException( "Problem with auth parameters") (verify, cert) = self.get_requests_verify_args() + # Turn off urllib3 warnings about insecure certs if we have + # explicitly configured requests to tell it we do not want + # cert verification + if not verify and urllib_exc is not None: + self.log.debug( + "Turning off SSL warnings for {cloud}:{region}" + " since verify=False".format( + cloud=self.name, region=self.region)) + warnings.filterwarnings( + 'ignore', category=urllib_exc.InsecureRequestWarning) self._keystone_session = session.Session( auth=self._auth, verify=verify, From ac51f4459106c10b0564747f36635779d40fadac Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 4 Nov 2015 13:33:41 -0500 Subject: [PATCH 180/365] Normalize int config values to string We have no things that expect actual int values. It's easy to write ints in yaml though - so normalize them to strings. Change-Id: I58cfdda697da9b7e030f4165ef8e60f4e9d00b66 --- os_client_config/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index af9dc3b48..eac4db610 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -235,6 +235,8 @@ class OpenStackConfig(object): key = key.replace('-', '_') if isinstance(value, dict): new_config[key] = self._normalize_keys(value) + elif isinstance(value, int): + new_config[key] = str(value) else: new_config[key] = value return new_config From 1bf09410d191b872b975634f8cf3f8847865b546 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 4 Nov 2015 18:28:41 -0500 Subject: [PATCH 181/365] Use requestsexceptions for urllib squelching The code to deal with this properly is quite sharable and we should not care. Use requestsexceptions from the Infra team to handle it. Change-Id: Ie20a3e1b2d8d18a4a76b34219cf12510cb1cda98 Depends-On: I52249b6d2fe04c49a9f4ed139d7625c890309ca8 --- os_client_config/cloud_config.py | 15 +++------------ requirements.txt | 1 + 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 17941c970..5e8d49293 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -16,19 +16,11 @@ import warnings from keystoneauth1 import plugin from keystoneauth1 import session +import requestsexceptions from os_client_config import _log from os_client_config import exceptions -# Importing these for later but not disabling for now -try: - from requests.packages.urllib3 import exceptions as urllib_exc -except ImportError: - try: - from urllib3 import exceptions as urllib_exc - except ImportError: - urllib_exc = None - class CloudConfig(object): def __init__(self, name, region, config, @@ -146,13 +138,12 @@ class CloudConfig(object): # Turn off urllib3 warnings about insecure certs if we have # explicitly configured requests to tell it we do not want # cert verification - if not verify and urllib_exc is not None: + if not verify: self.log.debug( "Turning off SSL warnings for {cloud}:{region}" " since verify=False".format( cloud=self.name, region=self.region)) - warnings.filterwarnings( - 'ignore', category=urllib_exc.InsecureRequestWarning) + requestsexceptions.squelch_warnings(insecure_requests=not verify) self._keystone_session = session.Session( auth=self._auth, verify=verify, diff --git a/requirements.txt b/requirements.txt index db0b6354a..3c32ced99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ PyYAML>=3.1.0 appdirs>=1.3.0 keystoneauth1>=1.0.0 +requestsexceptions>=1.1.1 # Apache-2.0 From 588be0126307a6b6dd0582ea546c73f05f23b919 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 5 Nov 2015 02:55:50 -0500 Subject: [PATCH 182/365] Dont turn bools into strings It turns out that requests does not like that. Change-Id: I206be8107f5cfaaa7dc7f34ab0b0764e0dc3fb0d --- os_client_config/config.py | 2 ++ os_client_config/tests/base.py | 1 + 2 files changed, 3 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index eac4db610..1bb7999f4 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -235,6 +235,8 @@ class OpenStackConfig(object): key = key.replace('-', '_') if isinstance(value, dict): new_config[key] = self._normalize_keys(value) + elif isinstance(value, bool): + new_config[key] = value elif isinstance(value, int): new_config[key] = str(value) else: diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 36c3bfb27..67c80f254 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -149,6 +149,7 @@ class TestCase(base.BaseTestCase): self.assertIsNone(cc.cloud) self.assertIn('username', cc.auth) self.assertEqual('testuser', cc.auth['username']) + self.assertFalse(cc.config['image_api_use_tasks']) self.assertTrue('project_name' in cc.auth or 'project_id' in cc.auth) if 'project_name' in cc.auth: self.assertEqual('testproject', cc.auth['project_name']) From 059d314ba44fd52f16436bf104b1a476df77bbdb Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 5 Nov 2015 02:11:23 -0500 Subject: [PATCH 183/365] Update network api version in defaults.json neutronclient expects 2.0, not 2 - but we'd not noticed because all of the consumption of client constructors were bypasing the version arg for neutron. In a follow-up patch we'll fix that, but the value in defaults.json is just straight wrong. Change-Id: I684c4ef063193355c6cf936d4f18576db919762b --- os_client_config/defaults.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index 9239d0fe8..d88ad0488 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -11,7 +11,7 @@ "image_api_use_tasks": false, "image_api_version": "2", "image_format": "qcow2", - "network_api_version": "2", + "network_api_version": "2.0", "object_api_version": "1", "orchestration_api_version": "1", "secgroup_source": "neutron", From d373969b98cd1db9188085dc2cd0b7754c687ff6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 5 Nov 2015 02:12:40 -0500 Subject: [PATCH 184/365] Don't assume pass_version_arg=False for network Excitingly, one of the places where we hard-code a workaround for a differently behaving client is not needed. The Client in neutron that takes a version arg is in neutronclient.neutron.client. Also, pass the version arg as positional when we pass it, as some of the clients (neutron) require it, and all accept it. Change-Id: Ifcbaab782173b95a678af0b2792a1194b198b687 --- os_client_config/cloud_config.py | 16 ++++++++-------- os_client_config/tests/test_cloud_config.py | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 17941c970..8e0b96336 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -238,10 +238,8 @@ class CloudConfig(object): interface_key = 'interface' else: interface_key = 'endpoint_type' - if service_key == 'network': - pass_version_arg = False - constructor_args = dict( + constructor_kwargs = dict( session=self.get_session(), service_name=self.get_service_name(service_key), service_type=self.get_service_type(service_key), @@ -254,13 +252,15 @@ class CloudConfig(object): # they necessarily have glanceclient installed from glanceclient.common import utils as glance_utils endpoint, version = glance_utils.strip_version(endpoint) - constructor_args['endpoint'] = endpoint - constructor_args.update(kwargs) - constructor_args[interface_key] = interface + constructor_kwargs['endpoint'] = endpoint + constructor_kwargs.update(kwargs) + constructor_kwargs[interface_key] = interface + constructor_args = [] if pass_version_arg: version = self.get_api_version(service_key) - constructor_args['version'] = version - return client_class(**constructor_args) + constructor_args.append(version) + + return client_class(*constructor_args, **constructor_kwargs) def _get_swift_client(self, client_class, **kwargs): session = self.get_session() diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index f5ceac7b7..080f1001d 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -24,7 +24,7 @@ from os_client_config.tests import base fake_config_dict = {'a': 1, 'os_b': 2, 'c': 3, 'os_c': 4} fake_services_dict = { - 'compute_api_version': 2, + 'compute_api_version': '2', 'compute_endpoint': 'http://compute.example.com', 'compute_region_name': 'region-bl', 'interface': 'public', @@ -139,7 +139,7 @@ class TestCloudConfig(base.TestCase): self.assertEqual('region-al', cc.get_region_name('image')) self.assertEqual('region-bl', cc.get_region_name('compute')) self.assertEqual(None, cc.get_api_version('image')) - self.assertEqual(2, cc.get_api_version('compute')) + self.assertEqual('2', cc.get_api_version('compute')) self.assertEqual('mage', cc.get_service_type('image')) self.assertEqual('compute', cc.get_service_type('compute')) self.assertEqual('http://compute.example.com', @@ -235,7 +235,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('image', mock_client) mock_client.assert_called_with( - version='2', + '2', service_name=None, endpoint='http://example.com', region_name='region-al', @@ -255,6 +255,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('network', mock_client) mock_client.assert_called_with( + '2.0', endpoint_type='public', region_name='region-al', service_type='network', @@ -271,7 +272,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('compute', mock_client) mock_client.assert_called_with( - version=2, + '2', endpoint_type='public', region_name='region-al', service_type='compute', From b8874ffc98488500fd4b573f7fa6196d3997e84e Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Thu, 5 Nov 2015 10:26:10 -0500 Subject: [PATCH 185/365] Convert floats to string We do it for int values, so floats should be covered, too. Change-Id: I1b98f7887ad421596b3bfdf2f6ad871a7d7f6bb2 --- os_client_config/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1bb7999f4..fd28c6910 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -239,6 +239,8 @@ class OpenStackConfig(object): new_config[key] = value elif isinstance(value, int): new_config[key] = str(value) + elif isinstance(value, float): + new_config[key] = str(value) else: new_config[key] = value return new_config From 2339243e66b676325c84693e7f8199d476199203 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 5 Nov 2015 17:00:33 -0500 Subject: [PATCH 186/365] Add method to get a mounted session from config Getting a session is great, but sometimes you need a thing called an "adapter" which takes 5 parameters which are all already contained in the config that you used to get the session. Change-Id: Id4e418cd04ae81540d9898f7b2e959b974f355d2 --- os_client_config/__init__.py | 17 +++++++++++++++++ os_client_config/cloud_config.py | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index d5fd36cb6..00e6ff514 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -13,3 +13,20 @@ # under the License. from os_client_config.config import OpenStackConfig # noqa + + +def simple_client(service_key, cloud=None, region_name=None): + """Simple wrapper function. It has almost no features. + + This will get you a raw requests Session Adapter that is mounted + on the given service from the keystone service catalog. If you leave + off cloud and region_name, it will assume that you've got env vars + set, but if you give them, it'll use clouds.yaml as you'd expect. + + This function is deliberately simple. It has no flexibility. If you + want flexibility, you can make a cloud config object and call + get_session_client on it. This function is to make it easy to poke + at OpenStack REST APIs with a properly configured keystone session. + """ + return OpenStackConfig().get_one_cloud( + cloud=cloud, region_name=region_name).get_session_client('compute') diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 8e0b96336..7cab1acd4 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -14,6 +14,7 @@ import warnings +from keystoneauth1 import adapter from keystoneauth1 import plugin from keystoneauth1 import session @@ -160,6 +161,28 @@ class CloudConfig(object): timeout=self.config['api_timeout']) return self._keystone_session + def get_session_client(self, service_key): + """Return a prepped requests adapter for a given service. + + This is useful for making direct requests calls against a + 'mounted' endpoint. That is, if you do: + + client = get_session_client('compute') + + then you can do: + + client.get('/flavors') + + and it will work like you think. + """ + + return adapter.Adapter( + session=self.get_session(), + service_type=self.get_service_type(service_key), + service_name=self.get_service_name(service_key), + interface=self.get_interface(service_key), + region_name=self.region) + def get_session_endpoint(self, service_key): """Return the endpoint from config or the catalog. From 5b993208b97a459429bcb5f6fb852372c576cdaf Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 6 Nov 2015 06:39:34 -0500 Subject: [PATCH 187/365] Return cache settings as numbers not strings While we're at it, let's also put in some tests to ensure that we're processing data types properly. Change-Id: I0442d234e8422a58738612b2da114f61cc9afc5c --- os_client_config/cloud_config.py | 10 ++++++---- os_client_config/config.py | 6 +++--- os_client_config/tests/base.py | 11 ++++++++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 8e0b96336..79a6ebff1 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -299,15 +299,17 @@ class CloudConfig(object): if self._openstack_config: return self._openstack_config.get_cache_expiration() - def get_cache_resource_expiration(self, resource): + def get_cache_resource_expiration(self, resource, default=None): """Get expiration time for a resource :param resource: Name of the resource type + :param default: Default value to return if not found (optional, + defaults to None) - :returns: Expiration time for the resource type or None + :returns: Expiration time for the resource type as float or default """ if self._openstack_config: expiration = self._openstack_config.get_cache_expiration() if resource not in expiration: - return None - return expiration[resource] + return default + return float(expiration[resource]) diff --git a/os_client_config/config.py b/os_client_config/config.py index fd28c6910..8d2a2ee76 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -246,13 +246,13 @@ class OpenStackConfig(object): return new_config def get_cache_expiration_time(self): - return self._cache_expiration_time + return int(self._cache_expiration_time) def get_cache_interval(self): - return self._cache_expiration_time + return self.get_cache_expiration_time() def get_cache_max_age(self): - return self._cache_expiration_time + return self.get_cache_expiration_time() def get_cache_path(self): return self._cache_path diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 67c80f254..3d94e2581 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -39,6 +39,13 @@ VENDOR_CONF = { } } USER_CONF = { + 'cache': { + 'max_age': '1', + 'expiration': { + 'server': 5, + 'image': '7', + }, + }, 'client': { 'force_ipv4': True, }, @@ -104,7 +111,6 @@ USER_CONF = { 'region_name': 'test-region', } }, - 'cache': {'max_age': 1}, } NO_CONF = { 'cache': {'max_age': 1}, @@ -155,3 +161,6 @@ class TestCase(base.BaseTestCase): self.assertEqual('testproject', cc.auth['project_name']) elif 'project_id' in cc.auth: self.assertEqual('testproject', cc.auth['project_id']) + self.assertEqual(cc.get_cache_expiration_time(), 1) + self.assertEqual(cc.get_cache_resource_expiration('server'), 5.0) + self.assertEqual(cc.get_cache_resource_expiration('image'), 7.0) From 13b6fbabeb27a346c91a9fb3a1fb86c7adbd2779 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 6 Nov 2015 09:04:12 -0500 Subject: [PATCH 188/365] Work around a bug in keystoneclient constructor keystoneclient bug #1513839 means that even when you proerly pass a Session (like we do) to the discovery constructor (what you'd be calling if you were passing "pass_version_arg = True") ksc fails because you didn't send a URL. We _HAVE_ an appropriate URL that we can pull from the Session. So until ksc learns how to pull the URL from the Session itself, do it for them. Change-Id: I38eb4cfa750fab5196b86989c2cd498d41bf37ac --- os_client_config/cloud_config.py | 11 +++++- os_client_config/tests/test_cloud_config.py | 37 +++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 79a6ebff1..37029f1bb 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -258,7 +258,16 @@ class CloudConfig(object): constructor_args = [] if pass_version_arg: version = self.get_api_version(service_key) - constructor_args.append(version) + if service_key == 'identity': + # keystoneclient takes version as a tuple. + version = tuple(str(float(version)).split('.')) + constructor_kwargs['version'] = version + # Workaround for bug#1513839 + if 'endpoint' not in constructor_kwargs: + endpoint = self.get_session_endpoint('identity') + constructor_kwargs['endpoint'] = endpoint + else: + constructor_args.append(version) return client_class(*constructor_args, **constructor_kwargs) diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 080f1001d..deaec4d35 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -278,3 +278,40 @@ class TestCloudConfig(base.TestCase): service_type='compute', session=mock.ANY, service_name=None) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_identity(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('identity', mock_client) + mock_client.assert_called_with( + version=('2', '0'), + endpoint='http://example.com/v2', + endpoint_type='admin', + region_name='region-al', + service_type='identity', + session=mock.ANY, + service_name='locks') + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_identity_v3(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + config_dict['identity_api_version'] = '3' + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('identity', mock_client) + mock_client.assert_called_with( + version=('3', '0'), + endpoint='http://example.com', + endpoint_type='admin', + region_name='region-al', + service_type='identity', + session=mock.ANY, + service_name='locks') From 59ad0ed00bfa485b5f0d24f5edb4e5b20cc7603a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 8 Nov 2015 18:21:46 -0500 Subject: [PATCH 189/365] Add default API version for magnum service shade is adding magnum support, so we should probably grow a default value for the container_api_version. Change-Id: I2551f2921f10ba109ccb52301cc8ad23c5f81c46 --- os_client_config/defaults.json | 1 + 1 file changed, 1 insertion(+) diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index d88ad0488..d1b66a318 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -1,6 +1,7 @@ { "auth_type": "password", "baremetal_api_version": "1", + "container_api_version": "1", "compute_api_version": "2", "database_api_version": "1.0", "disable_vendor_agent": {}, From ce7d716ffc5867d6cb7f73cbca6bbc4470499fb9 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 8 Nov 2015 18:24:24 -0500 Subject: [PATCH 190/365] Remove unneeded workaround for ksc It turns out keystoneclient can take string input and does the same transform we're doing here, it's just not documented. Remove the workaround on our side as it's unneccesary. Change-Id: Ie46945f7d96e3d65004cd19823b3be989e1d18a7 --- os_client_config/cloud_config.py | 6 +----- os_client_config/tests/test_cloud_config.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 37029f1bb..30dc27f6a 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -259,15 +259,11 @@ class CloudConfig(object): if pass_version_arg: version = self.get_api_version(service_key) if service_key == 'identity': - # keystoneclient takes version as a tuple. - version = tuple(str(float(version)).split('.')) - constructor_kwargs['version'] = version # Workaround for bug#1513839 if 'endpoint' not in constructor_kwargs: endpoint = self.get_session_endpoint('identity') constructor_kwargs['endpoint'] = endpoint - else: - constructor_args.append(version) + constructor_args.append(version) return client_class(*constructor_args, **constructor_kwargs) diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index deaec4d35..1b98b8aee 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -289,7 +289,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('identity', mock_client) mock_client.assert_called_with( - version=('2', '0'), + '2.0', endpoint='http://example.com/v2', endpoint_type='admin', region_name='region-al', @@ -308,7 +308,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('identity', mock_client) mock_client.assert_called_with( - version=('3', '0'), + '3', endpoint='http://example.com', endpoint_type='admin', region_name='region-al', From c90de1f691a27d4f434d948f1b77db02c23162a4 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 9 Nov 2015 09:07:06 -0500 Subject: [PATCH 191/365] Workaround for int value with verbose_level python-openstackclient uses an int value for verbose level which the stringification patch broke. As a quick fix, special case verbose_level, as fixing it properly might take more than 5 minutes. Change-Id: Ie12a40d3d3e400b3ec2103d7a58c4902fb10fc2d Closes-Bug: #1513919 --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 8d2a2ee76..f439d5f90 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -237,7 +237,7 @@ class OpenStackConfig(object): new_config[key] = self._normalize_keys(value) elif isinstance(value, bool): new_config[key] = value - elif isinstance(value, int): + elif isinstance(value, int) and key != 'verbose_level': new_config[key] = str(value) elif isinstance(value, float): new_config[key] = str(value) From 397da54db11b5a5506c24dc9af60a34665bbd953 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 9 Nov 2015 09:43:09 -0500 Subject: [PATCH 192/365] Workaround a dispute between osc and neutronclient python-openstackclient wants network_api_version to be 2. neutronclient wants it to be 2.0. There is a patch to OSC to make it understand both, but in the mean time, let's unbreak people. Change-Id: I4d8f187d1302c5bcfa246e017e6c6d8af9c3f733 --- os_client_config/cloud_config.py | 4 ++++ os_client_config/defaults.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 37029f1bb..bace53ef7 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -258,6 +258,10 @@ class CloudConfig(object): constructor_args = [] if pass_version_arg: version = self.get_api_version(service_key) + # Temporary workaround while we wait for python-openstackclient + # to be able to handle 2.0 which is what neutronclient expects + if service_key == 'network' and version == '2': + version = '2.0' if service_key == 'identity': # keystoneclient takes version as a tuple. version = tuple(str(float(version)).split('.')) diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index d88ad0488..9239d0fe8 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -11,7 +11,7 @@ "image_api_use_tasks": false, "image_api_version": "2", "image_format": "qcow2", - "network_api_version": "2.0", + "network_api_version": "2", "object_api_version": "1", "orchestration_api_version": "1", "secgroup_source": "neutron", From 506d6e8ffefdda4d31e1b33686008e9e610beca2 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Tue, 10 Nov 2015 17:16:02 -0500 Subject: [PATCH 193/365] Fix JSON schema We incorrectly refered to things with _api_ in the name. Change-Id: I9130ae0f72484957d0a8943eceabcd6a6bc21c91 --- os_client_config/vendor-schema.json | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/os_client_config/vendor-schema.json b/os_client_config/vendor-schema.json index e3fb57359..6c57ba4bc 100644 --- a/os_client_config/vendor-schema.json +++ b/os_client_config/vendor-schema.json @@ -61,87 +61,87 @@ "enum": [ "neutron", "nova", "None" ], "default": "neutron" }, - "compute_api_service_name": { + "compute_service_name": { "name": "Compute API Service Name", "description": "Compute API Service Name", "type": "string" }, - "database_api_service_name": { + "database_service_name": { "name": "Database API Service Name", "description": "Database API Service Name", "type": "string" }, - "dns_api_service_name": { + "dns_service_name": { "name": "DNS API Service Name", "description": "DNS API Service Name", "type": "string" }, - "identity_api_service_name": { + "identity_service_name": { "name": "Identity API Service Name", "description": "Identity API Service Name", "type": "string" }, - "image_api_service_name": { + "image_service_name": { "name": "Image API Service Name", "description": "Image API Service Name", "type": "string" }, - "volume_api_service_name": { + "volume_service_name": { "name": "Volume API Service Name", "description": "Volume API Service Name", "type": "string" }, - "network_api_service_name": { + "network_service_name": { "name": "Network API Service Name", "description": "Network API Service Name", "type": "string" }, - "object_api_service_name": { + "object_service_name": { "name": "Object Storage API Service Name", "description": "Object Storage API Service Name", "type": "string" }, - "baremetal_api_service_name": { + "baremetal_service_name": { "name": "Baremetal API Service Name", "description": "Baremetal API Service Name", "type": "string" }, - "compute_api_service_type": { + "compute_service_type": { "name": "Compute API Service Type", "description": "Compute API Service Type", "type": "string" }, - "database_api_service_type": { + "database_service_type": { "name": "Database API Service Type", "description": "Database API Service Type", "type": "string" }, - "dns_api_service_type": { + "dns_service_type": { "name": "DNS API Service Type", "description": "DNS API Service Type", "type": "string" }, - "identity_api_service_type": { + "identity_service_type": { "name": "Identity API Service Type", "description": "Identity API Service Type", "type": "string" }, - "image_api_service_type": { + "image_service_type": { "name": "Image API Service Type", "description": "Image API Service Type", "type": "string" }, - "volume_api_service_type": { + "volume_service_type": { "name": "Volume API Service Type", "description": "Volume API Service Type", "type": "string" }, - "network_api_service_type": { + "network_service_type": { "name": "Network API Service Type", "description": "Network API Service Type", "type": "string" }, - "object_api_service_type": { + "object_service_type": { "name": "Object Storage API Service Type", "description": "Object Storage API Service Type", "type": "string" From f5e1b859c8547865deb0ef8241c51ea4a00a7f97 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 12 Nov 2015 09:43:05 -0500 Subject: [PATCH 194/365] Add support for legacy envvar prefixes In trying to move the legacy clients to do their config via os-client-config, many of them had prefixes like NOVA_ or GLANCE_ in their shells. It's pretty easy to support optionally passing those in so that the transition to OCC is less of a change and we can treat the deprecation cycle of those features as a different topic. Change-Id: Ic819c0790989fa034db03d83535ee7b70aaacc3a --- os_client_config/config.py | 20 ++++++++++++-------- os_client_config/tests/test_environ.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index f439d5f90..8d92c8fe7 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -81,21 +81,24 @@ def get_boolean(value): return False -def _get_os_environ(): +def _get_os_environ(envvar_prefix=None): ret = defaults.get_defaults() + if not envvar_prefix: + # This makes the or below be OS_ or OS_ which is a no-op + envvar_prefix = 'OS_' environkeys = [k for k in os.environ.keys() - if k.startswith('OS_') + if k.startswith('OS_') or k.startswith(envvar_prefix) and not k.startswith('OS_TEST') # infra CI var and not k.startswith('OS_STD') # infra CI var ] + for k in environkeys: + newkey = k.split('_', 1)[-1].lower() + ret[newkey] = os.environ[k] # If the only environ key is region name, don't make a cloud, because # it's being used as a cloud selector if not environkeys or ( - len(environkeys) == 1 and 'OS_REGION_NAME' in environkeys): + len(environkeys) == 1 and 'region_name' in ret): return None - for k in environkeys: - newkey = k[3:].lower() - ret[newkey] = os.environ[k] return ret @@ -115,7 +118,8 @@ def _auth_update(old_dict, new_dict): class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, - override_defaults=None, force_ipv4=None): + override_defaults=None, force_ipv4=None, + envvar_prefix=None): self._config_files = config_files or CONFIG_FILES self._vendor_files = vendor_files or VENDOR_FILES @@ -173,7 +177,7 @@ class OpenStackConfig(object): # make an envvars cloud self.default_cloud = os.environ.pop('OS_CLOUD', None) - envvars = _get_os_environ() + envvars = _get_os_environ(envvar_prefix=envvar_prefix) if envvars: self.cloud_config['clouds'][self.envvar_key] = envvars diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 7f284c5eb..1e804fc3c 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -31,6 +31,8 @@ class TestEnviron(base.TestCase): fixtures.EnvironmentVariable('OS_USERNAME', 'testuser')) self.useFixture( fixtures.EnvironmentVariable('OS_PROJECT_NAME', 'testproject')) + self.useFixture( + fixtures.EnvironmentVariable('NOVA_PROJECT_ID', 'testnova')) def test_get_one_cloud(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], @@ -66,6 +68,22 @@ class TestEnviron(base.TestCase): self._assert_cloud_details(cc) self.assertNotIn('auth_url', cc.config) self.assertIn('auth_url', cc.config['auth']) + self.assertNotIn('project_id', cc.config['auth']) + self.assertNotIn('auth_url', cc.config) + cc = c.get_one_cloud('_test-cloud_') + self._assert_cloud_details(cc) + cc = c.get_one_cloud('_test_cloud_no_vendor') + self._assert_cloud_details(cc) + + def test_environ_prefix(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + envvar_prefix='NOVA_') + cc = c.get_one_cloud('envvars') + self._assert_cloud_details(cc) + self.assertNotIn('auth_url', cc.config) + self.assertIn('auth_url', cc.config['auth']) + self.assertIn('project_id', cc.config['auth']) self.assertNotIn('auth_url', cc.config) cc = c.get_one_cloud('_test-cloud_') self._assert_cloud_details(cc) From 3e76af913ae4a29ba1e4dd9fb4cf3b5ca109f79c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 14 Nov 2015 12:10:09 -0500 Subject: [PATCH 195/365] Refactor per-service key making The key for swift is object-store, but the key in the config dict would be object_store_api_version, so the key concatenation would not work. In fixing that, refactor out the creation of the keys so that the concatenation and transformation needed always happens. Change-Id: Ic095912bfc84f13ef8b11f312303a517289e0441 --- os_client_config/cloud_config.py | 22 +++++++++++++-------- os_client_config/tests/test_cloud_config.py | 19 ++++++++++++++++++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 3f46c6ecd..32fd3b6d5 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -23,6 +23,14 @@ from os_client_config import _log from os_client_config import exceptions +def _make_key(key, service_type): + if not service_type: + return key + else: + service_type = service_type.lower().replace('-', '_') + return "_".join([service_type, key]) + + class CloudConfig(object): def __init__(self, name, region, config, force_ipv4=False, auth_plugin=None, @@ -89,32 +97,30 @@ class CloudConfig(object): return self.config['auth'] def get_interface(self, service_type=None): + key = _make_key('interface', service_type) interface = self.config.get('interface') - if not service_type: - return interface - key = '{service_type}_interface'.format(service_type=service_type) return self.config.get(key, interface) def get_region_name(self, service_type=None): if not service_type: return self.region - key = '{service_type}_region_name'.format(service_type=service_type) + key = _make_key('region_name', service_type) return self.config.get(key, self.region) def get_api_version(self, service_type): - key = '{service_type}_api_version'.format(service_type=service_type) + key = _make_key('api_version', service_type) return self.config.get(key, None) def get_service_type(self, service_type): - key = '{service_type}_service_type'.format(service_type=service_type) + key = _make_key('service_type', service_type) return self.config.get(key, service_type) def get_service_name(self, service_type): - key = '{service_type}_service_name'.format(service_type=service_type) + key = _make_key('service_name', service_type) return self.config.get(key, None) def get_endpoint(self, service_type): - key = '{service_type}_endpoint'.format(service_type=service_type) + key = _make_key('endpoint', service_type) return self.config.get(key, None) @property diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 1b98b8aee..be2b8fe4a 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -225,6 +225,25 @@ class TestCloudConfig(base.TestCase): auth_version='2.0', timeout=None) + def test_legacy_client_object_store_endpoint(self): + mock_client = mock.Mock() + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + config_dict['object_store_endpoint'] = 'http://example.com/v2' + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('object-store', mock_client) + mock_client.assert_called_with( + preauthtoken=mock.ANY, + os_options={ + 'auth_token': mock.ANY, + 'region_name': 'region-al', + 'object_storage_url': 'http://example.com/v2' + }, + preauthurl='http://example.com/v2', + auth_version='2.0', + timeout=None) + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') def test_legacy_client_image(self, mock_get_session_endpoint): mock_client = mock.Mock() From 0aefe152af40436f532e3b37ff8ab602a0af6e6f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 14 Nov 2015 12:12:39 -0500 Subject: [PATCH 196/365] Fix name of the object-store api key There is only one value for this, and it's not consumed in the process of making a swift client, so the chances that anyone set it to something else are pretty much nil. However, for completeness we should make it the right name, as "object-store" is the service key name for swift, not "object". Change-Id: I395c1c44a2f50996b61dff22e07149b8dd13eda9 --- os_client_config/defaults.json | 2 +- os_client_config/schema.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index 208641aff..eb8162e4e 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -13,7 +13,7 @@ "image_api_version": "2", "image_format": "qcow2", "network_api_version": "2", - "object_api_version": "1", + "object_store_api_version": "1", "orchestration_api_version": "1", "secgroup_source": "neutron", "volume_api_version": "1" diff --git a/os_client_config/schema.json b/os_client_config/schema.json index dfd1f4a9b..8110d58e9 100644 --- a/os_client_config/schema.json +++ b/os_client_config/schema.json @@ -87,7 +87,7 @@ "default": "2", "type": "string" }, - "object_api_version": { + "object_store_api_version": { "name": "Object Storage API Version", "description": "Object Storage API Version", "default": "1", @@ -114,7 +114,7 @@ "image_format", "interface", "network_api_version", - "object_api_version", + "object_store_api_version", "secgroup_source", "volume_api_version" ] From 10e96bcb7bed2ca8eaf0185b46dbb882bd8b2b7c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 19 Nov 2015 17:15:08 -0500 Subject: [PATCH 197/365] Only pass timeout to swift if we have a value swiftclient does not have the built-in None detection that keystoneauth has, so we have to do it ourselves. Change-Id: I3fcf60dd2f3350045b824899ac48b02809452a4b --- os_client_config/cloud_config.py | 6 ++-- os_client_config/tests/test_cloud_config.py | 38 +++++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 32fd3b6d5..3174f60c6 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -297,7 +297,7 @@ class CloudConfig(object): endpoint = self.get_session_endpoint(service_key='object-store') if not endpoint: return None - return client_class( + swift_kwargs = dict( preauthurl=endpoint, preauthtoken=token, auth_version=self.get_api_version('identity'), @@ -305,8 +305,10 @@ class CloudConfig(object): auth_token=token, object_storage_url=endpoint, region_name=self.get_region_name()), - timeout=self.api_timeout, ) + if self.config['api_timeout'] is not None: + swift_kwargs['timeout'] = float(self.config['api_timeout']) + return client_class(**swift_kwargs) def get_cache_expiration_time(self): if self._openstack_config: diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index be2b8fe4a..45d262f30 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -168,6 +168,18 @@ class TestCloudConfig(base.TestCase): auth=mock.ANY, verify=True, cert=None, timeout=None) + @mock.patch.object(ksa_session, 'Session') + def test_get_session_with_timeout(self, mock_session): + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + config_dict['api_timeout'] = 9 + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_session() + mock_session.assert_called_with( + auth=mock.ANY, + verify=True, cert=None, timeout=9) + @mock.patch.object(ksa_session, 'Session') def test_override_session_endpoint(self, mock_session): config_dict = defaults.get_defaults() @@ -214,6 +226,27 @@ class TestCloudConfig(base.TestCase): cc = cloud_config.CloudConfig( "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('object-store', mock_client) + mock_client.assert_called_with( + preauthtoken=mock.ANY, + os_options={ + 'auth_token': mock.ANY, + 'region_name': 'region-al', + 'object_storage_url': 'http://example.com/v2' + }, + preauthurl='http://example.com/v2', + auth_version='2.0') + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_object_store_timeout( + self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + config_dict['api_timeout'] = 9 + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( preauthtoken=mock.ANY, os_options={ @@ -223,7 +256,7 @@ class TestCloudConfig(base.TestCase): }, preauthurl='http://example.com/v2', auth_version='2.0', - timeout=None) + timeout=9.0) def test_legacy_client_object_store_endpoint(self): mock_client = mock.Mock() @@ -241,8 +274,7 @@ class TestCloudConfig(base.TestCase): 'object_storage_url': 'http://example.com/v2' }, preauthurl='http://example.com/v2', - auth_version='2.0', - timeout=None) + auth_version='2.0') @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') def test_legacy_client_image(self, mock_get_session_endpoint): From 9c59002116e755d9ac7f14b11c3774c6848d4a3c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 20 Nov 2015 12:10:16 -0500 Subject: [PATCH 198/365] Fix lack of parenthesis around boolean logic The legacy envvar prefix support broke a workaround for use of OS_ envvars in test cases. This is only currently a problem for shade functional tests, but it IS a bug. Change-Id: Ia0cbb4e2ea7ce6eeeea36533e057bd53a830d44c --- os_client_config/config.py | 2 +- os_client_config/tests/test_environ.py | 40 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 8d92c8fe7..5f7c402ae 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -87,7 +87,7 @@ def _get_os_environ(envvar_prefix=None): # This makes the or below be OS_ or OS_ which is a no-op envvar_prefix = 'OS_' environkeys = [k for k in os.environ.keys() - if k.startswith('OS_') or k.startswith(envvar_prefix) + if (k.startswith('OS_') or k.startswith(envvar_prefix)) and not k.startswith('OS_TEST') # infra CI var and not k.startswith('OS_STD') # infra CI var ] diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 1e804fc3c..0ff800fd4 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -111,3 +111,43 @@ class TestEnviron(base.TestCase): vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud('_test-cloud_') self._assert_cloud_details(cc) + + +class TestEnvvars(base.TestCase): + + def test_no_envvars(self): + self.useFixture( + fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + self.assertRaises( + exceptions.OpenStackConfigException, c.get_one_cloud, 'envvars') + + def test_test_envvars(self): + self.useFixture( + fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) + self.useFixture( + fixtures.EnvironmentVariable('OS_STDERR_CAPTURE', 'True')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + self.assertRaises( + exceptions.OpenStackConfigException, c.get_one_cloud, 'envvars') + + def test_have_envvars(self): + self.useFixture( + fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) + self.useFixture( + fixtures.EnvironmentVariable('OS_USERNAME', 'user')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('envvars') + self.assertEqual(cc.config['auth']['username'], 'user') + + def test_old_envvars(self): + self.useFixture( + fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + envvar_prefix='NOVA_') + cc = c.get_one_cloud('envvars') + self.assertEqual(cc.config['auth']['username'], 'nova') From b17bbcdef9acfaf6e2b47671c9982d3378045961 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 22 Nov 2015 13:16:44 -0500 Subject: [PATCH 199/365] Add support for secure.yaml file for auth info Almost nothing in clouds.yaml is secret, but the file has to be treated as if it were because of the passwords or other secrets contained in it. This makes it difficult to put clouds.yaml into a public or broadly accessible config repository. Add support for having a second optional file, secure.yaml, which can contain any value you can put in clouds.yaml and which will be overlayed on top of clouds.yaml values. Most people probably do not need this, but for folks with complex cloud configs with teams of people working on them, this reduces the amount of things that have to be managed by the privileged system. Change-Id: I631d826588b0a0b1f36244caa7982dd42d9eb498 --- README.rst | 28 ++++++++++++++++++++++ os_client_config/config.py | 33 +++++++++++++++++++++++++- os_client_config/tests/base.py | 12 +++++++++- os_client_config/tests/test_config.py | 21 ++++++++++------ os_client_config/tests/test_environ.py | 14 +++++++---- 5 files changed, 95 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index f16bbc091..0bdc18290 100644 --- a/README.rst +++ b/README.rst @@ -145,6 +145,34 @@ as a result of a chosen plugin need to go into the auth dict. For password auth, this includes `auth_url`, `username` and `password` as well as anything related to domains, projects and trusts. +Splitting Secrets +----------------- + +In some scenarios, such as configuragtion managment controlled environments, +it might be eaiser to have secrets in one file and non-secrets in another. +This is fully supported via an optional file `secure.yaml` which follows all +the same location rules as `clouds.yaml`. It can contain anything you put +in `clouds.yaml` and will take precedence over anything in the `clouds.yaml` +file. + +:: + + # clouds.yaml + clouds: + internap: + profile: internap + auth: + username: api-55f9a00fb2619 + project_name: inap-17037 + regions: + - ams01 + - nyj01 + # secure.yaml + clouds: + internap: + auth: + password: XXXXXXXXXXXXXXXXX + SSL Settings ------------ diff --git a/os_client_config/config.py b/os_client_config/config.py index 5f7c402ae..c12b25a40 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -51,6 +51,11 @@ CONFIG_FILES = [ for d in CONFIG_SEARCH_PATH for s in YAML_SUFFIXES + JSON_SUFFIXES ] +SECURE_FILES = [ + os.path.join(d, 'secure' + s) + for d in CONFIG_SEARCH_PATH + for s in YAML_SUFFIXES + JSON_SUFFIXES +] VENDOR_FILES = [ os.path.join(d, 'clouds-public' + s) for d in CONFIG_SEARCH_PATH @@ -102,6 +107,20 @@ def _get_os_environ(envvar_prefix=None): return ret +def _merge_clouds(old_dict, new_dict): + """Like dict.update, except handling nested dicts.""" + ret = old_dict.copy() + for (k, v) in new_dict.items(): + if isinstance(v, dict): + if k in ret: + ret[k] = _merge_clouds(ret[k], v) + else: + ret[k] = v.copy() + else: + ret[k] = v + return ret + + def _auth_update(old_dict, new_dict): """Like dict.update, except handling the nested dict called auth.""" for (k, v) in new_dict.items(): @@ -119,20 +138,29 @@ class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, override_defaults=None, force_ipv4=None, - envvar_prefix=None): + envvar_prefix=None, secure_files=None): self._config_files = config_files or CONFIG_FILES + self._secure_files = secure_files or SECURE_FILES self._vendor_files = vendor_files or VENDOR_FILES config_file_override = os.environ.pop('OS_CLIENT_CONFIG_FILE', None) if config_file_override: self._config_files.insert(0, config_file_override) + secure_file_override = os.environ.pop('OS_CLIENT_SECURE_FILE', None) + if secure_file_override: + self._secure_files.insert(0, secure_file_override) + self.defaults = defaults.get_defaults() if override_defaults: self.defaults.update(override_defaults) # First, use a config file if it exists where expected self.config_filename, self.cloud_config = self._load_config_file() + _, secure_config = self._load_secure_file() + if secure_config: + self.cloud_config = _merge_clouds( + self.cloud_config, secure_config) if not self.cloud_config: self.cloud_config = {'clouds': {}} @@ -220,6 +248,9 @@ class OpenStackConfig(object): def _load_config_file(self): return self._load_yaml_json_file(self._config_files) + def _load_secure_file(self): + return self._load_yaml_json_file(self._secure_files) + def _load_vendor_file(self): return self._load_yaml_json_file(self._vendor_files) diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 3d94e2581..33a868d33 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -64,7 +64,6 @@ USER_CONF = { 'auth': { 'auth_url': 'http://example.com/v2', 'username': 'testuser', - 'password': 'testpass', 'project_name': 'testproject', }, 'region-name': 'test-region', @@ -112,6 +111,15 @@ USER_CONF = { } }, } +SECURE_CONF = { + 'clouds': { + '_test_cloud_no_vendor': { + 'auth': { + 'password': 'testpass', + }, + } + } +} NO_CONF = { 'cache': {'max_age': 1}, } @@ -135,6 +143,7 @@ class TestCase(base.BaseTestCase): tdir = self.useFixture(fixtures.TempDir()) conf['cache']['path'] = tdir.path self.cloud_yaml = _write_yaml(conf) + self.secure_yaml = _write_yaml(SECURE_CONF) self.vendor_yaml = _write_yaml(VENDOR_CONF) self.no_yaml = _write_yaml(NO_CONF) @@ -155,6 +164,7 @@ class TestCase(base.BaseTestCase): self.assertIsNone(cc.cloud) self.assertIn('username', cc.auth) self.assertEqual('testuser', cc.auth['username']) + self.assertEqual('testpass', cc.auth['password']) self.assertFalse(cc.config['image_api_use_tasks']) self.assertTrue('project_name' in cc.auth or 'project_id' in cc.auth) if 'project_name' in cc.auth: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index d225b7c17..aff8c6d25 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -30,7 +30,8 @@ class TestConfig(base.TestCase): def test_get_all_clouds(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) clouds = c.get_all_clouds() # We add one by hand because the regions cloud is going to exist # twice since it has two regions in it @@ -74,7 +75,8 @@ class TestConfig(base.TestCase): def test_get_one_cloud_with_config_files(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.secure_yaml]) self.assertIsInstance(c.cloud_config, dict) self.assertIn('cache', c.cloud_config) self.assertIsInstance(c.cloud_config['cache'], dict) @@ -129,7 +131,8 @@ class TestConfig(base.TestCase): def test_fallthrough(self): c = config.OpenStackConfig(config_files=[self.no_yaml], - vendor_files=[self.no_yaml]) + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) for k in os.environ.keys(): if k.startswith('OS_'): self.useFixture(fixtures.EnvironmentVariable(k)) @@ -137,7 +140,8 @@ class TestConfig(base.TestCase): def test_prefer_ipv6_true(self): c = config.OpenStackConfig(config_files=[self.no_yaml], - vendor_files=[self.no_yaml]) + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) cc = c.get_one_cloud(cloud='defaults', validate=False) self.assertTrue(cc.prefer_ipv6) @@ -155,7 +159,8 @@ class TestConfig(base.TestCase): def test_force_ipv4_false(self): c = config.OpenStackConfig(config_files=[self.no_yaml], - vendor_files=[self.no_yaml]) + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) cc = c.get_one_cloud(cloud='defaults', validate=False) self.assertFalse(cc.force_ipv4) @@ -166,7 +171,8 @@ class TestConfig(base.TestCase): self.assertEqual('testpass', cc.auth['password']) def test_get_cloud_names(self): - c = config.OpenStackConfig(config_files=[self.cloud_yaml]) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + secure_files=[self.no_yaml]) self.assertEqual( ['_test-cloud-domain-id_', '_test-cloud-int-project_', @@ -177,7 +183,8 @@ class TestConfig(base.TestCase): ], sorted(c.get_cloud_names())) c = config.OpenStackConfig(config_files=[self.no_yaml], - vendor_files=[self.no_yaml]) + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) for k in os.environ.keys(): if k.startswith('OS_'): self.useFixture(fixtures.EnvironmentVariable(k)) diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 0ff800fd4..b75db1c61 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -29,6 +29,8 @@ class TestEnviron(base.TestCase): fixtures.EnvironmentVariable('OS_AUTH_URL', 'https://example.com')) self.useFixture( fixtures.EnvironmentVariable('OS_USERNAME', 'testuser')) + self.useFixture( + fixtures.EnvironmentVariable('OS_PASSWORD', 'testpass')) self.useFixture( fixtures.EnvironmentVariable('OS_PROJECT_NAME', 'testproject')) self.useFixture( @@ -57,13 +59,15 @@ class TestEnviron(base.TestCase): self.useFixture( fixtures.EnvironmentVariable('OS_PREFER_IPV6', 'false')) c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.secure_yaml]) cc = c.get_one_cloud('_test-cloud_') self.assertFalse(cc.prefer_ipv6) def test_environ_exists(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.secure_yaml]) cc = c.get_one_cloud('envvars') self._assert_cloud_details(cc) self.assertNotIn('auth_url', cc.config) @@ -78,7 +82,8 @@ class TestEnviron(base.TestCase): def test_environ_prefix(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml], - envvar_prefix='NOVA_') + envvar_prefix='NOVA_', + secure_files=[self.secure_yaml]) cc = c.get_one_cloud('envvars') self._assert_cloud_details(cc) self.assertNotIn('auth_url', cc.config) @@ -92,7 +97,8 @@ class TestEnviron(base.TestCase): def test_get_one_cloud_with_config_files(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.secure_yaml]) self.assertIsInstance(c.cloud_config, dict) self.assertIn('cache', c.cloud_config) self.assertIsInstance(c.cloud_config['cache'], dict) From 0ea32e7dc011373c61e4d79d710a09e4ae404a9e Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 15 Nov 2015 11:34:41 -0500 Subject: [PATCH 200/365] Handle cinder v2 It turns out that cinder v2 has a service_type of volumev2 because nobody thought of the children. But that's ok - we actually care about user experience around here. SO - take the sane approach and return service_type = volumev2 if service_type == volume and volume_api_version == 2. This way user code can all safely say "please give me the endpoint for the volume service" and can use the api_version parameter to specify which version they want. We should all possess righteous indignation about this patch. Change-Id: I15fc5ddd92345d78b6928f11a8d77cecd0427f7d --- os_client_config/cloud_config.py | 8 ++++++++ os_client_config/tests/test_cloud_config.py | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 3174f60c6..18ea4c1eb 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -113,6 +113,14 @@ class CloudConfig(object): def get_service_type(self, service_type): key = _make_key('service_type', service_type) + # Cinder did an evil thing where they defined a second service + # type in the catalog. Of course, that's insane, so let's hide this + # atrocity from the as-yet-unsullied eyes of our users. + # Of course, if the user requests a volumev2, that structure should + # still work. + if (service_type == 'volume' and + self.get_api_version(service_type).startswith('2')): + service_type = 'volumev2' return self.config.get(key, service_type) def get_service_name(self, service_type): diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 45d262f30..9e683d1da 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -31,6 +31,7 @@ fake_services_dict = { 'image_service_type': 'mage', 'identity_interface': 'admin', 'identity_service_name': 'locks', + 'volume_api_version': '1', 'auth': {'password': 'hunter2', 'username': 'AzureDiamond'}, } @@ -128,7 +129,7 @@ class TestCloudConfig(base.TestCase): def test_getters(self): cc = cloud_config.CloudConfig("test1", "region-al", fake_services_dict) - self.assertEqual(['compute', 'identity', 'image'], + self.assertEqual(['compute', 'identity', 'image', 'volume'], sorted(cc.get_services())) self.assertEqual({'password': 'hunter2', 'username': 'AzureDiamond'}, cc.get_auth_args()) @@ -142,6 +143,8 @@ class TestCloudConfig(base.TestCase): self.assertEqual('2', cc.get_api_version('compute')) self.assertEqual('mage', cc.get_service_type('image')) self.assertEqual('compute', cc.get_service_type('compute')) + self.assertEqual('1', cc.get_api_version('volume')) + self.assertEqual('volume', cc.get_service_type('volume')) self.assertEqual('http://compute.example.com', cc.get_endpoint('compute')) self.assertEqual(None, @@ -149,6 +152,11 @@ class TestCloudConfig(base.TestCase): self.assertEqual(None, cc.get_service_name('compute')) self.assertEqual('locks', cc.get_service_name('identity')) + def test_volume_override(self): + cc = cloud_config.CloudConfig("test1", "region-al", fake_services_dict) + cc.config['volume_api_version'] = '2' + self.assertEqual('volumev2', cc.get_service_type('volume')) + def test_get_session_no_auth(self): config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) From cc92c92870be48d6eae57869c4989125fe2eb8e4 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 25 Nov 2015 09:37:45 -0800 Subject: [PATCH 201/365] Add BHS1 to OVH Change-Id: I0ef175edccbbc3e24803d02ab6809cfe1a68e0e8 --- doc/source/vendor-support.rst | 1 + os_client_config/vendors/ovh.json | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index a9dd5ef8c..90fd31fad 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -177,6 +177,7 @@ https://auth.cloud.ovh.net/v2.0 ============== ================ Region Name Human Name ============== ================ +BHS1 Beauharnois, QC SBG1 Strassbourg, FR GRA1 Gravelines, FR ============== ================ diff --git a/os_client_config/vendors/ovh.json b/os_client_config/vendors/ovh.json index cfd234b64..032741f83 100644 --- a/os_client_config/vendors/ovh.json +++ b/os_client_config/vendors/ovh.json @@ -5,6 +5,7 @@ "auth_url": "https://auth.cloud.ovh.net/v2.0" }, "regions": [ + "BHS1", "GRA1", "SBG1" ], From 026a17c9eb9d8ebad8c56f8d1b7946bd4694519e Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Wed, 2 Dec 2015 11:05:29 +0800 Subject: [PATCH 202/365] Remove optional keystoneauth1 imports keystoneauth1 is now a hard dependency of os-client-config so there is no way that this should not be importable. Change-Id: I20901623e8b29f50d7ab1ed956472a4b1eda51bf --- os_client_config/config.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 5f7c402ae..19cfa3598 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -18,10 +18,7 @@ import os import warnings import appdirs -try: - from keystoneauth1 import loading -except ImportError: - loading = None +from keystoneauth1 import loading import yaml from os_client_config import cloud_config @@ -657,27 +654,23 @@ class OpenStackConfig(object): # compatible behaviour config = self.auth_config_hook(config) - if loading: - if validate: - try: - loader = self._get_auth_loader(config) - config = self._validate_auth(config, loader) - auth_plugin = loader.load_from_options(**config['auth']) - except Exception as e: - # We WANT the ksa exception normally - # but OSC can't handle it right now, so we try deferring - # to ksc. If that ALSO fails, it means there is likely - # a deeper issue, so we assume the ksa error was correct - auth_plugin = None - try: - config = self._validate_auth_ksc(config) - except Exception: - raise e - else: + if validate: + try: + loader = self._get_auth_loader(config) + config = self._validate_auth(config, loader) + auth_plugin = loader.load_from_options(**config['auth']) + except Exception as e: + # We WANT the ksa exception normally + # but OSC can't handle it right now, so we try deferring + # to ksc. If that ALSO fails, it means there is likely + # a deeper issue, so we assume the ksa error was correct auth_plugin = None + try: + config = self._validate_auth_ksc(config) + except Exception: + raise e else: auth_plugin = None - config = self._validate_auth_ksc(config) # If any of the defaults reference other values, we need to expand for (key, value) in config.items(): From eea460d5917a1536025d66ce3e2b3f9094d46e7c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 3 Dec 2015 07:34:23 -0800 Subject: [PATCH 203/365] Make sure that cloud always has a name If we don't ask for a cloud, and we fall through to the envvars cloud or the defaults cloud, the cloud that is returned winds up not being named - even though we know what cloud it is. Set the name of the cloud we're working with. This is important for the next patch, where we need to peek at the config to get some default values, but in a fallthrough case we do not know which cloud to request. Change-Id: Ie56e490d4384f2d680450bc956e4b7b5b8099f0e --- os_client_config/config.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 19cfa3598..ea3f6a1c3 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -177,6 +177,8 @@ class OpenStackConfig(object): envvars = _get_os_environ(envvar_prefix=envvar_prefix) if envvars: self.cloud_config['clouds'][self.envvar_key] = envvars + if not self.default_cloud: + self.default_cloud = self.envvar_key # Finally, fall through and make a cloud that starts with defaults # because we need somewhere to put arguments, and there are neither @@ -184,6 +186,7 @@ class OpenStackConfig(object): if not self.cloud_config['clouds']: self.cloud_config = dict( clouds=dict(defaults=dict(self.defaults))) + self.default_cloud = 'defaults' self._cache_expiration_time = 0 self._cache_path = CACHE_PATH @@ -604,14 +607,14 @@ class OpenStackConfig(object): on missing required auth parameters """ - if cloud is None and self.default_cloud: - cloud = self.default_cloud - - if cloud is None and self.envvar_key in self.get_cloud_names(): - cloud = self.envvar_key - args = self._fix_args(kwargs, argparse=argparse) + if cloud is None: + if 'cloud' in args: + cloud = args['cloud'] + else: + cloud = self.default_cloud + if 'region_name' not in args or args['region_name'] is None: args['region_name'] = self._get_region(cloud) From 6aecb87e7f2cb2c031bf9c89564ea0b46d387c79 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 6 Dec 2015 10:35:08 -0500 Subject: [PATCH 204/365] Update vexxhost to Identity v3 There is a discovery URL for vexxhost for keystone v3. Also, there is a new vexxhost domain for it. Also, vexxhost has DNS running designate v1. And make the region list a list of one region, because there is a second region coming soon. Change-Id: Ie72c19976646f41c713124659e69725df59e1580 --- doc/source/vendor-support.rst | 5 ++++- os_client_config/vendors/vexxhost.json | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 90fd31fad..d7af6b913 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -271,10 +271,13 @@ gd1 Guangdong vexxhost -------- -http://auth.api.thenebulacloud.com:5000/v2.0/ +http://auth.vexxhost.net ============== ================ Region Name Human Name ============== ================ ca-ymq-1 Montreal ============== ================ + +* DNS API Version is 1 +* Identity API Version is 3 diff --git a/os_client_config/vendors/vexxhost.json b/os_client_config/vendors/vexxhost.json index 25911cae1..dd683be86 100644 --- a/os_client_config/vendors/vexxhost.json +++ b/os_client_config/vendors/vexxhost.json @@ -2,9 +2,13 @@ "name": "vexxhost", "profile": { "auth": { - "auth_url": "http://auth.api.thenebulacloud.com:5000/v2.0/" + "auth_url": "http://auth.vexxhost.net" }, - "region_name": "ca-ymq-1", + "regions": [ + "ca-ymq-1" + ], + "dns_api_version": "1", + "identity_api_version": "3", "floating_ip_source": "None" } } From ed2f34b06a7d581fb5fdd9811e3f8a7f748a2ce4 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 4 Nov 2015 09:22:17 -0500 Subject: [PATCH 205/365] Add method for registering argparse options keystoneauth knows about a bunch of argparse options that users from a command line will want. We do a good job of processing them once they've been collected, but an os-client-config user doesn't have a great way to make sure that they register all of the options, especially when once considers that you really want to peek at the args to see which auth plugin has been selected so that the right arguments can be registered and displayed. Depends-On: Ifea90b981044009c3642b268dd639a703df1ef05 Change-Id: Ic196f65f89b3ccf92ebec39564f5eaefe8a4ae4b --- README.rst | 23 +++++ os_client_config/config.py | 124 ++++++++++++++++++++++++++ os_client_config/tests/test_config.py | 115 ++++++++++++++++++++++++ requirements.txt | 2 +- 4 files changed, 263 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0bdc18290..d8fde5924 100644 --- a/README.rst +++ b/README.rst @@ -296,3 +296,26 @@ Or, get all of the clouds. cloud_config = os_client_config.OpenStackConfig().get_all_clouds() for cloud in cloud_config: print(cloud.name, cloud.region, cloud.config) + +argparse +-------- + +If you're using os-client-config from a program that wants to process +command line options, there is a registration function to register the +arguments that both os-client-config and keystoneauth know how to deal +with - as well as a consumption argument. + +:: + + import argparse + import sys + + import os_client_config + + cloud_config = os_client_config.OpenStackConfig() + parser = argparse.ArgumentParser() + cloud_config.register_argparse_arguments(parser, sys.argv) + + options = parser.parse_args() + + cloud = cloud_config.get_one_cloud(argparse=options) diff --git a/os_client_config/config.py b/os_client_config/config.py index dff637a10..48aa0e2d0 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -13,11 +13,14 @@ # under the License. +# alias because we already had an option named argparse +import argparse as argparse_mod import json import os import warnings import appdirs +from keystoneauth1 import adapter from keystoneauth1 import loading import yaml @@ -245,6 +248,9 @@ class OpenStackConfig(object): self._cache_expiration = cache_settings.get( 'expiration', self._cache_expiration) + # Flag location to hold the peeked value of an argparse timeout value + self._argv_timeout = False + def _load_config_file(self): return self._load_yaml_json_file(self._config_files) @@ -451,6 +457,94 @@ class OpenStackConfig(object): cloud['auth_type'] = 'password' return cloud + def register_argparse_arguments(self, parser, argv, service_keys=[]): + """Register all of the common argparse options needed. + + Given an argparse parser, register the keystoneauth Session arguments, + the keystoneauth Auth Plugin Options and os-cloud. Also, peek in the + argv to see if all of the auth plugin options should be registered + or merely the ones already configured. + :param argparse.ArgumentParser: parser to attach argparse options to + :param list argv: the arguments provided to the application + :param string service_keys: Service or list of services this argparse + should be specialized for, if known. + The first item in the list will be used + as the default value for service_type + (optional) + + :raises exceptions.OpenStackConfigException if an invalid auth-type + is requested + """ + + local_parser = argparse_mod.ArgumentParser(add_help=False) + + for p in (parser, local_parser): + p.add_argument( + '--os-cloud', + metavar='', + default=os.environ.get('OS_CLOUD', None), + help='Named cloud to connect to') + + # we need to peek to see if timeout was actually passed, since + # the keystoneauth declaration of it has a default, which means + # we have no clue if the value we get is from the ksa default + # for from the user passing it explicitly. We'll stash it for later + local_parser.add_argument('--timeout', metavar='') + + # Peek into the future and see if we have an auth-type set in + # config AND a cloud set, so that we know which command line + # arguments to register and show to the user (the user may want + # to say something like: + # openstack --os-cloud=foo --os-oidctoken=bar + # although I think that user is the cause of my personal pain + options, _args = local_parser.parse_known_args(argv) + if options.timeout: + self._argv_timeout = True + + # validate = False because we're not _actually_ loading here + # we're only peeking, so it's the wrong time to assert that + # the rest of the arguments given are invalid for the plugin + # chosen (for instance, --help may be requested, so that the + # user can see what options he may want to give + cloud = self.get_one_cloud(argparse=options, validate=False) + default_auth_type = cloud.config['auth_type'] + + try: + loading.register_auth_argparse_arguments( + parser, argv, default=default_auth_type) + except Exception: + # Hidiing the keystoneauth exception because we're not actually + # loading the auth plugin at this point, so the error message + # from it doesn't actually make sense to os-client-config users + options, _args = parser.parse_known_args(argv) + plugin_names = loading.get_available_plugin_names() + raise exceptions.OpenStackConfigException( + "An invalid auth-type was specified: {auth_type}." + " Valid choices are: {plugin_names}.".format( + auth_type=options.os_auth_type, + plugin_names=",".join(plugin_names))) + + if service_keys: + primary_service = service_keys[0] + else: + primary_service = None + loading.register_session_argparse_arguments(parser) + adapter.register_adapter_argparse_arguments( + parser, service_type=primary_service) + for service_key in service_keys: + # legacy clients have un-prefixed api-version options + parser.add_argument( + '--{service_key}-api-version'.format( + service_key=service_key.replace('_', '-'), + help=argparse_mod.SUPPRESS)) + adapter.register_service_adapter_argparse_arguments( + parser, service_type=service_key) + + # Backwards compat options for legacy clients + parser.add_argument('--http-timeout', help=argparse_mod.SUPPRESS) + parser.add_argument('--os-endpoint-type', help=argparse_mod.SUPPRESS) + parser.add_argument('--endpoint-type', help=argparse_mod.SUPPRESS) + def _fix_backwards_interface(self, cloud): new_cloud = {} for key in cloud.keys(): @@ -461,6 +555,30 @@ class OpenStackConfig(object): new_cloud[target_key] = cloud[key] return new_cloud + def _fix_backwards_api_timeout(self, cloud): + new_cloud = {} + # requests can only have one timeout, which means that in a single + # cloud there is no point in different timeout values. However, + # for some reason many of the legacy clients decided to shove their + # service name in to the arg name for reasons surpassin sanity. If + # we find any values that are not api_timeout, overwrite api_timeout + # with the value + service_timeout = None + for key in cloud.keys(): + if key.endswith('timeout') and not ( + key == 'timeout' or key == 'api_timeout'): + service_timeout = cloud[key] + else: + new_cloud[key] = cloud[key] + if service_timeout is not None: + new_cloud['api_timeout'] = service_timeout + # The common argparse arg from keystoneauth is called timeout, but + # os-client-config expects it to be called api_timeout + if self._argv_timeout: + if 'timeout' in new_cloud and new_cloud['timeout']: + new_cloud['api_timeout'] = new_cloud.pop('timeout') + return new_cloud + def get_all_clouds(self): clouds = [] @@ -671,6 +789,12 @@ class OpenStackConfig(object): else: config[key] = val + # These backwards compat values are only set via argparse. If it's + # there, it's because it was passed in explicitly, and should win + config = self._fix_backwards_api_timeout(config) + if 'endpoint_type' in config: + config['interface'] = config.pop('endpoint_type') + for key in BOOL_KEYS: if key in config: if type(config[key]) is not bool: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index aff8c6d25..c9318fcd9 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -17,6 +17,7 @@ import copy import os import fixtures +import testtools import yaml from os_client_config import cloud_config @@ -341,6 +342,120 @@ class TestConfigArgparse(base.TestCase): self.assertDictEqual({'compute_api_version': 1}, fixed_args) + def test_register_argparse_cloud(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + parser = argparse.ArgumentParser() + c.register_argparse_arguments(parser, []) + opts, _remain = parser.parse_known_args(['--os-cloud', 'foo']) + self.assertEqual(opts.os_cloud, 'foo') + + def test_register_argparse_bad_plugin(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + parser = argparse.ArgumentParser() + self.assertRaises( + exceptions.OpenStackConfigException, + c.register_argparse_arguments, + parser, ['--os-auth-type', 'foo']) + + def test_register_argparse_not_password(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + parser = argparse.ArgumentParser() + args = [ + '--os-auth-type', 'v3token', + '--os-token', 'some-secret', + ] + c.register_argparse_arguments(parser, args) + opts, _remain = parser.parse_known_args(args) + self.assertEqual(opts.os_token, 'some-secret') + + def test_register_argparse_password(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + parser = argparse.ArgumentParser() + args = [ + '--os-password', 'some-secret', + ] + c.register_argparse_arguments(parser, args) + opts, _remain = parser.parse_known_args(args) + self.assertEqual(opts.os_password, 'some-secret') + with testtools.ExpectedException(AttributeError): + opts.os_token + + def test_register_argparse_service_type(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + parser = argparse.ArgumentParser() + args = [ + '--os-service-type', 'network', + '--os-endpoint-type', 'admin', + '--http-timeout', '20', + ] + c.register_argparse_arguments(parser, args) + opts, _remain = parser.parse_known_args(args) + self.assertEqual(opts.os_service_type, 'network') + self.assertEqual(opts.os_endpoint_type, 'admin') + self.assertEqual(opts.http_timeout, '20') + with testtools.ExpectedException(AttributeError): + opts.os_network_service_type + cloud = c.get_one_cloud(argparse=opts, verify=False) + self.assertEqual(cloud.config['service_type'], 'network') + self.assertEqual(cloud.config['interface'], 'admin') + self.assertEqual(cloud.config['api_timeout'], '20') + self.assertNotIn('http_timeout', cloud.config) + + def test_register_argparse_network_service_type(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + parser = argparse.ArgumentParser() + args = [ + '--os-endpoint-type', 'admin', + '--network-api-version', '4', + ] + c.register_argparse_arguments(parser, args, ['network']) + opts, _remain = parser.parse_known_args(args) + self.assertEqual(opts.os_service_type, 'network') + self.assertEqual(opts.os_endpoint_type, 'admin') + self.assertEqual(opts.os_network_service_type, None) + self.assertEqual(opts.os_network_api_version, None) + self.assertEqual(opts.network_api_version, '4') + cloud = c.get_one_cloud(argparse=opts, verify=False) + self.assertEqual(cloud.config['service_type'], 'network') + self.assertEqual(cloud.config['interface'], 'admin') + self.assertEqual(cloud.config['network_api_version'], '4') + self.assertNotIn('http_timeout', cloud.config) + + def test_register_argparse_network_service_types(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + parser = argparse.ArgumentParser() + args = [ + '--os-compute-service-name', 'cloudServers', + '--os-network-service-type', 'badtype', + '--os-endpoint-type', 'admin', + '--network-api-version', '4', + ] + c.register_argparse_arguments( + parser, args, ['compute', 'network', 'volume']) + opts, _remain = parser.parse_known_args(args) + self.assertEqual(opts.os_network_service_type, 'badtype') + self.assertEqual(opts.os_compute_service_type, None) + self.assertEqual(opts.os_volume_service_type, None) + self.assertEqual(opts.os_service_type, 'compute') + self.assertEqual(opts.os_compute_service_name, 'cloudServers') + self.assertEqual(opts.os_endpoint_type, 'admin') + self.assertEqual(opts.os_network_api_version, None) + self.assertEqual(opts.network_api_version, '4') + cloud = c.get_one_cloud(argparse=opts, verify=False) + self.assertEqual(cloud.config['service_type'], 'compute') + self.assertEqual(cloud.config['network_service_type'], 'badtype') + self.assertEqual(cloud.config['interface'], 'admin') + self.assertEqual(cloud.config['network_api_version'], '4') + self.assertNotIn('volume_service_type', cloud.config) + self.assertNotIn('http_timeout', cloud.config) + class TestConfigDefault(base.TestCase): diff --git a/requirements.txt b/requirements.txt index 3c32ced99..1531be808 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ # process, which may cause wedges in the gate later. PyYAML>=3.1.0 appdirs>=1.3.0 -keystoneauth1>=1.0.0 +keystoneauth1>=2.1.0 requestsexceptions>=1.1.1 # Apache-2.0 From 5beaeef2c3140f84b2e5a57a789460d4db9ff766 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 3 Dec 2015 11:26:12 -0600 Subject: [PATCH 206/365] Add simple helper function for client construction Often times you don't want to take advantage of all the flexibility, you simple want the basic works-like-it-should thing. Add a warpper around get_legacy_client to do tht one thing. Change-Id: I086dc4a8e762d4e8e56e01cabe2386577f2ceec8 --- README.rst | 31 +++++++++++++++++++++++++++++++ os_client_config/__init__.py | 23 +++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/README.rst b/README.rst index d8fde5924..156c7607a 100644 --- a/README.rst +++ b/README.rst @@ -319,3 +319,34 @@ with - as well as a consumption argument. options = parser.parse_args() cloud = cloud_config.get_one_cloud(argparse=options) + +Constructing OpenStack Client objects +------------------------------------- + +If all you want to do is get a Client object from a python-*client library, +and you want it to do all the normal things related to clouds.yaml, `OS_` +environment variables, a hepler function is provided. + +:: + + import argparse + + from novaclient import client + import os_client_config + + nova = os_client_config.make_client('compute', client.Client) + +If you want to do the same thing but also support command line parsing. + +:: + + import argparse + + from novaclient import client + import os_client_config + + nova = os_client_config.make_client( + 'compute', client.Client, options=argparse.ArgumentParser()) + +If you want to get fancier than that in your python, then the rest of the +API is avaiable to you. But often times, you just want to do the one thing. diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index 00e6ff514..ac585f248 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import sys + from os_client_config.config import OpenStackConfig # noqa @@ -30,3 +32,24 @@ def simple_client(service_key, cloud=None, region_name=None): """ return OpenStackConfig().get_one_cloud( cloud=cloud, region_name=region_name).get_session_client('compute') + + +def make_client(service_key, constructor, options=None, **kwargs): + """Simple wrapper for getting a client instance from a client lib. + + OpenStack Client Libraries all have a fairly consistent constructor + interface which os-client-config supports. In the simple case, there + is one and only one right way to construct a client object. If as a user + you don't want to do fancy things, just use this. It honors OS_ environment + variables and clouds.yaml - and takes as **kwargs anything you'd expect + to pass in. + """ + config = OpenStackConfig() + if options: + config.register_argparse_options(options, sys.argv, service_key) + parsed_options = options.parse_args(sys.argv) + else: + parsed_options = None + + cloud_config = config.get_one_cloud(options=parsed_options, **kwargs) + return cloud_config.get_legacy_client(service_key, constructor) From b90f53bbf45c67fd2139b3f75c8f25bff0d3cfeb Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 6 Dec 2015 21:49:29 -0500 Subject: [PATCH 207/365] Updated README to clarify legacy client usage Also, update it to use code-block - which makes things look much nicer. Change-Id: I930ab63a5d159cf4cea27b4e2c4d6fd933de04fc --- README.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 156c7607a..585dda9b9 100644 --- a/README.rst +++ b/README.rst @@ -29,7 +29,7 @@ Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type for trove set -:: +.. code-block:: bash export OS_DATABASE_SERVICE_TYPE=rax:database @@ -56,7 +56,7 @@ Service specific settings, like the nova service type, are set with the default service type as a prefix. For instance, to set a special service_type for trove (because you're using Rackspace) set: -:: +.. code-block:: yaml database_service_type: 'rax:database' @@ -85,7 +85,7 @@ look in an OS specific config dir An example config file is probably helpful: -:: +.. code-block:: yaml clouds: mordred: @@ -155,7 +155,7 @@ the same location rules as `clouds.yaml`. It can contain anything you put in `clouds.yaml` and will take precedence over anything in the `clouds.yaml` file. -:: +.. code-block:: yaml # clouds.yaml clouds: @@ -209,7 +209,7 @@ that the resource should never expire. and presents the cache information so that your various applications that are connecting to OpenStack can share a cache should you desire. -:: +.. code-block:: yaml cache: class: dogpile.cache.pylibmc @@ -242,7 +242,7 @@ caused it to not actually function. In that case, there is a config option you can set to unbreak you `force_ipv4`, or `OS_FORCE_IPV4` boolean environment variable. -:: +.. code-block:: yaml client: force_ipv4: true @@ -270,7 +270,7 @@ Usage The simplest and least useful thing you can do is: -:: +.. code-block:: python python -m os_client_config.config @@ -279,7 +279,7 @@ it from python, which is much more likely what you want to do, things like: Get a named cloud. -:: +.. code-block:: python import os_client_config @@ -289,7 +289,7 @@ Get a named cloud. Or, get all of the clouds. -:: +.. code-block:: python import os_client_config @@ -305,7 +305,7 @@ command line options, there is a registration function to register the arguments that both os-client-config and keystoneauth know how to deal with - as well as a consumption argument. -:: +.. code-block:: python import argparse import sys @@ -320,14 +320,14 @@ with - as well as a consumption argument. cloud = cloud_config.get_one_cloud(argparse=options) -Constructing OpenStack Client objects -------------------------------------- +Constructing Legacy Client objects +---------------------------------- If all you want to do is get a Client object from a python-*client library, and you want it to do all the normal things related to clouds.yaml, `OS_` environment variables, a hepler function is provided. -:: +.. code-block:: python import argparse @@ -338,7 +338,7 @@ environment variables, a hepler function is provided. If you want to do the same thing but also support command line parsing. -:: +.. code-block:: python import argparse From 8eced67abe20160fc3f20a7c76f01baae2dd1956 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 3 Dec 2015 13:28:00 -0600 Subject: [PATCH 208/365] Make client constructor optional Turns out we know the mapping of service name to constsructor, so we can try the import for the user without actually importing. Keep the argument though, because this method should be usable by just about any random openstack client lib. Also, because backwards compat. Change-Id: I7e9672e3bf61b8b7b92d55903f4596382f18b515 --- README.rst | 9 +++---- os_client_config/cloud_config.py | 41 ++++++++++++++++++++++++++++++- os_client_config/constructors.py | 28 +++++++++++++++++++++ os_client_config/constructos.json | 10 ++++++++ 4 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 os_client_config/constructors.py create mode 100644 os_client_config/constructos.json diff --git a/README.rst b/README.rst index 156c7607a..7541b9335 100644 --- a/README.rst +++ b/README.rst @@ -325,16 +325,16 @@ Constructing OpenStack Client objects If all you want to do is get a Client object from a python-*client library, and you want it to do all the normal things related to clouds.yaml, `OS_` -environment variables, a hepler function is provided. +environment variables, a hepler function is provided. The following +will get you a fully configured `novaclient` instance. :: import argparse - from novaclient import client import os_client_config - nova = os_client_config.make_client('compute', client.Client) + nova = os_client_config.make_client('compute') If you want to do the same thing but also support command line parsing. @@ -342,11 +342,10 @@ If you want to do the same thing but also support command line parsing. import argparse - from novaclient import client import os_client_config nova = os_client_config.make_client( - 'compute', client.Client, options=argparse.ArgumentParser()) + 'compute', options=argparse.ArgumentParser()) If you want to get fancier than that in your python, then the rest of the API is avaiable to you. But often times, you just want to do the one thing. diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 18ea4c1eb..6b3b5d9a0 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import importlib import warnings from keystoneauth1 import adapter @@ -20,9 +21,44 @@ from keystoneauth1 import session import requestsexceptions from os_client_config import _log +from os_client_config import constructors from os_client_config import exceptions +def _get_client(service_key): + class_mapping = constructors.get_constructor_mapping() + if service_key not in class_mapping: + raise exceptions.OpenStackConfigException( + "Service {service_key} is unkown. Please pass in a client" + " constructor or submit a patch to os-client-config".format( + service_key=service_key)) + mod_name, ctr_name = class_mapping[service_key].rsplit('.', 1) + lib_name = mod_name.split('.')[0] + try: + mod = importlib.import_module(mod_name) + except ImportError: + raise exceptions.OpenStackConfigException( + "Client for '{service_key}' was requested, but" + " {mod_name} was unable to be imported. Either import" + " the module yourself and pass the constructor in as an argument," + " or perhaps you do not have python-{lib_name} installed.".format( + service_key=service_key, + mod_name=mod_name, + lib_name=lib_name)) + try: + ctr = getattr(mod, ctr_name) + except AttributeError: + raise exceptions.OpenStackConfigException( + "Client for '{service_key}' was requested, but although" + " {mod_name} imported fine, the constructor at {fullname}" + " as not found. Please check your installation, we have no" + " clue what is wrong with your computer.".format( + service_key=service_key, + mod_name=mod_name, + fullname=class_mapping[service_key])) + return ctr + + def _make_key(key, service_type): if not service_type: return key @@ -217,7 +253,7 @@ class CloudConfig(object): return endpoint def get_legacy_client( - self, service_key, client_class, interface_key=None, + self, service_key, client_class=None, interface_key=None, pass_version_arg=True, **kwargs): """Return a legacy OpenStack client object for the given config. @@ -254,6 +290,9 @@ class CloudConfig(object): Client constructor, so this is in case anything additional needs to be passed in. """ + if not client_class: + client_class = _get_client(service_key) + # Because of course swift is different if service_key == 'object-store': return self._get_swift_client(client_class=client_class, **kwargs) diff --git a/os_client_config/constructors.py b/os_client_config/constructors.py new file mode 100644 index 000000000..e88ac92d6 --- /dev/null +++ b/os_client_config/constructors.py @@ -0,0 +1,28 @@ +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import os + +_json_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), 'constructors.json') +_class_mapping = None + + +def get_constructor_mapping(): + global _class_mapping + if not _class_mapping: + with open(_json_path, 'r') as json_file: + _class_mapping = json.load(json_file) + return _class_mapping diff --git a/os_client_config/constructos.json b/os_client_config/constructos.json new file mode 100644 index 000000000..d9ebf2c97 --- /dev/null +++ b/os_client_config/constructos.json @@ -0,0 +1,10 @@ +{ + "compute": "novaclient.client.Client", + "database": "troveclient.client.Client", + "identity": "keystoneclient.client.Client", + "image": "glanceclient.Client", + "network": "neutronclient.neutron.client.Client", + "object-store": "swiftclient.client.Connection", + "orchestration": "heatclient.client.Client", + "volume": "cinderclient.client.Client" +} From 1221ea7fca67c22c455b4aeae1e09c9ad7e928a7 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 4 Dec 2015 13:19:19 -0500 Subject: [PATCH 209/365] Fix a README typo - hepler is not actually a thing Change-Id: Ie8c267e75171b88bd3a1a3a684e85869e75843d7 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7541b9335..4edff9e14 100644 --- a/README.rst +++ b/README.rst @@ -325,7 +325,7 @@ Constructing OpenStack Client objects If all you want to do is get a Client object from a python-*client library, and you want it to do all the normal things related to clouds.yaml, `OS_` -environment variables, a hepler function is provided. The following +environment variables, a helper function is provided. The following will get you a fully configured `novaclient` instance. :: From d0c70cc96279d2bf24f30a501b9bf572e40f8e7a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 22 Nov 2015 10:55:46 -0500 Subject: [PATCH 210/365] Add support for generalized per-region settings Internap creates a public and a private network for each customer for each region on region activation. This means there is a per-region external network that the user may want to specify. Also, conoha has per-region auth-urls. Per-region config is still overridden by argparse or kwargs values. Change-Id: Ie2f3d2ca3ccbe7e3dd674983136b42c323544997 --- README.rst | 32 ++++++++++ os_client_config/config.py | 90 ++++++++++++++++++--------- os_client_config/tests/base.py | 17 ++++- os_client_config/tests/test_config.py | 50 ++++++++++++--- 4 files changed, 146 insertions(+), 43 deletions(-) diff --git a/README.rst b/README.rst index 9850c05ac..ced3b1821 100644 --- a/README.rst +++ b/README.rst @@ -265,6 +265,38 @@ environment variable. The above snippet will tell client programs to prefer returning an IPv4 address. +Per-region settings +------------------- + +Sometimes you have a cloud provider that has config that is common to the +cloud, but also with some things you might want to express on a per-region +basis. For instance, Internap provides a public and private network specific +to the user in each region, and putting the values of those networks into +config can make consuming programs more efficient. + +To support this, the region list can actually be a list of dicts, and any +setting that can be set at the cloud level can be overridden for that +region. + +:: + + clouds: + internap: + profile: internap + auth: + password: XXXXXXXXXXXXXXXXX + username: api-55f9a00fb2619 + project_name: inap-17037 + regions: + - name: ams01 + values: + external_network: inap-17037-WAN1654 + internal_network: inap-17037-LAN4820 + - name: nyj01 + values: + external_network: inap-17037-WAN7752 + internal_network: inap-17037-LAN6745 + Usage ----- diff --git a/os_client_config/config.py b/os_client_config/config.py index 48aa0e2d0..70989bfd7 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -15,6 +15,7 @@ # alias because we already had an option named argparse import argparse as argparse_mod +import copy import json import os import warnings @@ -121,8 +122,9 @@ def _merge_clouds(old_dict, new_dict): return ret -def _auth_update(old_dict, new_dict): +def _auth_update(old_dict, new_dict_source): """Like dict.update, except handling the nested dict called auth.""" + new_dict = copy.deepcopy(new_dict_source) for (k, v) in new_dict.items(): if k == 'auth': if k in old_dict: @@ -302,17 +304,29 @@ class OpenStackConfig(object): return self._cache_class def get_cache_arguments(self): - return self._cache_arguments.copy() + return copy.deepcopy(self._cache_arguments) def get_cache_expiration(self): - return self._cache_expiration.copy() + return copy.deepcopy(self._cache_expiration) + + def _expand_region_name(self, region_name): + return {'name': region_name, 'values': {}} + + def _expand_regions(self, regions): + ret = [] + for region in regions: + if isinstance(region, dict): + ret.append(copy.deepcopy(region)) + else: + ret.append(self._expand_region_name(region)) + return ret def _get_regions(self, cloud): if cloud not in self.cloud_config['clouds']: - return [''] + return [self._expand_region_name('')] config = self._normalize_keys(self.cloud_config['clouds'][cloud]) if 'regions' in config: - return config['regions'] + return self._expand_regions(config['regions']) elif 'region_name' in config: regions = config['region_name'].split(',') if len(regions) > 1: @@ -320,22 +334,39 @@ class OpenStackConfig(object): "Comma separated lists in region_name are deprecated." " Please use a yaml list in the regions" " parameter in {0} instead.".format(self.config_filename)) - return regions + return self._expand_regions(regions) else: # crappit. we don't have a region defined. new_cloud = dict() our_cloud = self.cloud_config['clouds'].get(cloud, dict()) self._expand_vendor_profile(cloud, new_cloud, our_cloud) if 'regions' in new_cloud and new_cloud['regions']: - return new_cloud['regions'] + return self._expand_regions(new_cloud['regions']) elif 'region_name' in new_cloud and new_cloud['region_name']: - return [new_cloud['region_name']] + return [self._expand_region_name(new_cloud['region_name'])] else: # Wow. We really tried - return [''] + return [self._expand_region_name('')] - def _get_region(self, cloud=None): - return self._get_regions(cloud)[0] + def _get_region(self, cloud=None, region_name=''): + if not cloud: + return self._expand_region_name(region_name) + + regions = self._get_regions(cloud) + if not region_name: + return regions[0] + + for region in regions: + if region['name'] == region_name: + return region + + raise exceptions.OpenStackConfigException( + 'Region {region_name} is not a valid region name for cloud' + ' {cloud}. Valid choices are {region_list}. Please note that' + ' region names are case sensitive.'.format( + region_name=region_name, + region_list=','.join([r['name'] for r in regions]), + cloud=cloud)) def get_cloud_names(self): return self.cloud_config['clouds'].keys() @@ -585,7 +616,9 @@ class OpenStackConfig(object): for cloud in self.get_cloud_names(): for region in self._get_regions(cloud): - clouds.append(self.get_one_cloud(cloud, region_name=region)) + if region: + clouds.append(self.get_one_cloud( + cloud, region_name=region['name'])) return clouds def _fix_args(self, args, argparse=None): @@ -764,30 +797,27 @@ class OpenStackConfig(object): else: cloud = self.default_cloud - if 'region_name' not in args or args['region_name'] is None: - args['region_name'] = self._get_region(cloud) - config = self._get_base_cloud_config(cloud) + # Get region specific settings + if 'region_name' not in args: + args['region_name'] = '' + region = self._get_region(cloud=cloud, region_name=args['region_name']) + args['region_name'] = region['name'] + region_args = copy.deepcopy(region['values']) + # Regions is a list that we can use to create a list of cloud/region # objects. It does not belong in the single-cloud dict - regions = config.pop('regions', None) - if regions and args['region_name'] not in regions: - raise exceptions.OpenStackConfigException( - 'Region {region_name} is not a valid region name for cloud' - ' {cloud}. Valid choices are {region_list}. Please note that' - ' region names are case sensitive.'.format( - region_name=args['region_name'], - region_list=','.join(regions), - cloud=cloud)) + config.pop('regions', None) # Can't just do update, because None values take over - for (key, val) in iter(args.items()): - if val is not None: - if key == 'auth' and config[key] is not None: - config[key] = _auth_update(config[key], val) - else: - config[key] = val + for arg_list in region_args, args: + for (key, val) in iter(arg_list.items()): + if val is not None: + if key == 'auth' and config[key] is not None: + config[key] = _auth_update(config[key], val) + else: + config[key] = val # These backwards compat values are only set via argparse. If it's # there, it's because it was passed in explicitly, and should win diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 33a868d33..6d9e093d3 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -16,6 +16,7 @@ # under the License. +import copy import os import tempfile @@ -96,8 +97,18 @@ USER_CONF = { 'auth_url': 'http://example.com/v2', }, 'regions': [ - 'region1', - 'region2', + { + 'name': 'region1', + 'values': { + 'external_network': 'region1-network', + } + }, + { + 'name': 'region2', + 'values': { + 'external_network': 'my-network', + } + } ], }, '_test_cloud_hyphenated': { @@ -139,7 +150,7 @@ class TestCase(base.BaseTestCase): super(TestCase, self).setUp() self.useFixture(fixtures.NestedTempfile()) - conf = dict(USER_CONF) + conf = copy.deepcopy(USER_CONF) tdir = self.useFixture(fixtures.TempDir()) conf['cache']['path'] = tdir.path self.cloud_yaml = _write_yaml(conf) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index c9318fcd9..a6a35ada9 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -226,6 +226,8 @@ class TestConfig(base.TestCase): new_config) with open(self.cloud_yaml) as fh: written_config = yaml.safe_load(fh) + # We write a cache config for testing + written_config['cache'].pop('path', None) self.assertEqual(written_config, resulting_config) @@ -239,18 +241,26 @@ class TestConfigArgparse(base.TestCase): username='user', password='password', project_name='project', - region_name='other-test-region', + region_name='region2', snack_type='cookie', ) self.options = argparse.Namespace(**self.args) + def test_get_one_cloud_bad_region_argparse(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + self.assertRaises( + exceptions.OpenStackConfigException, c.get_one_cloud, + cloud='_test-cloud_', argparse=self.options) + def test_get_one_cloud_argparse(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud(cloud='_test-cloud_', argparse=self.options) - self._assert_cloud_details(cc) - self.assertEqual(cc.region_name, 'other-test-region') + cc = c.get_one_cloud( + cloud='_test_cloud_regions', argparse=self.options) + self.assertEqual(cc.region_name, 'region2') self.assertEqual(cc.snack_type, 'cookie') def test_get_one_cloud_just_argparse(self): @@ -259,7 +269,7 @@ class TestConfigArgparse(base.TestCase): cc = c.get_one_cloud(argparse=self.options) self.assertIsNone(cc.cloud) - self.assertEqual(cc.region_name, 'other-test-region') + self.assertEqual(cc.region_name, 'region2') self.assertEqual(cc.snack_type, 'cookie') def test_get_one_cloud_just_kwargs(self): @@ -268,7 +278,7 @@ class TestConfigArgparse(base.TestCase): cc = c.get_one_cloud(**self.args) self.assertIsNone(cc.cloud) - self.assertEqual(cc.region_name, 'other-test-region') + self.assertEqual(cc.region_name, 'region2') self.assertEqual(cc.snack_type, 'cookie') def test_get_one_cloud_dash_kwargs(self): @@ -318,10 +328,10 @@ class TestConfigArgparse(base.TestCase): def test_get_one_cloud_bad_region_no_regions(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - - cc = c.get_one_cloud(cloud='_test-cloud_', region_name='bad_region') - self._assert_cloud_details(cc) - self.assertEqual(cc.region_name, 'bad_region') + self.assertRaises( + exceptions.OpenStackConfigException, + c.get_one_cloud, + cloud='_test-cloud_', region_name='bad_region') def test_get_one_cloud_no_argparse_region2(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], @@ -333,6 +343,26 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(cc.region_name, 'region2') self.assertIsNone(cc.snack_type) + def test_get_one_cloud_network(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud( + cloud='_test_cloud_regions', region_name='region1', argparse=None) + self._assert_cloud_details(cc) + self.assertEqual(cc.region_name, 'region1') + self.assertEqual('region1-network', cc.config['external_network']) + + def test_get_one_cloud_per_region_network(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud( + cloud='_test_cloud_regions', region_name='region2', argparse=None) + self._assert_cloud_details(cc) + self.assertEqual(cc.region_name, 'region2') + self.assertEqual('my-network', cc.config['external_network']) + def test_fix_env_args(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From f4237a809cccbbffce2233b1f283b36a9ebb75c1 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 9 Dec 2015 15:42:20 -0500 Subject: [PATCH 211/365] Add ceilometer constructor to known constructors In porting ospurge to use get_legacy_client, it became clear that the ceilometer client constructor was missing. Add it. Change-Id: I1102105b78574378c4f11064e21245b08513247b --- os_client_config/{constructos.json => constructors.json} | 1 + os_client_config/defaults.json | 1 + 2 files changed, 2 insertions(+) rename os_client_config/{constructos.json => constructors.json} (88%) diff --git a/os_client_config/constructos.json b/os_client_config/constructors.json similarity index 88% rename from os_client_config/constructos.json rename to os_client_config/constructors.json index d9ebf2c97..be4433920 100644 --- a/os_client_config/constructos.json +++ b/os_client_config/constructors.json @@ -3,6 +3,7 @@ "database": "troveclient.client.Client", "identity": "keystoneclient.client.Client", "image": "glanceclient.Client", + "metering": "ceilometerclient.client.Client", "network": "neutronclient.neutron.client.Client", "object-store": "swiftclient.client.Connection", "orchestration": "heatclient.client.Client", diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index eb8162e4e..6735b5535 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -12,6 +12,7 @@ "image_api_use_tasks": false, "image_api_version": "2", "image_format": "qcow2", + "metering_api_version": "2", "network_api_version": "2", "object_store_api_version": "1", "orchestration_api_version": "1", From 837ca712288ceffea5b54ceaeb349d6577f38360 Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Tue, 17 Nov 2015 15:17:55 -0500 Subject: [PATCH 212/365] Allow arbitrary client-specific options There are occasionally some client-specific things that would be handy to be able to configure about behaviors. For instance, the only config file that ansible's openstack inventory has is clouds.yaml. Rather than teaching os-client-config about such things, allow a pass-through config section. Apply key normalization to _'s like other configs, and merge the clouds and secure files so that the sections behave like other OCC config sections. Change-Id: If307e95006abf6e1efbbd77cfc99e5fdfed6c80a --- os_client_config/config.py | 13 +++++++++++++ os_client_config/tests/base.py | 4 ++++ os_client_config/tests/test_config.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 70989bfd7..ab3a003c6 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -253,6 +253,19 @@ class OpenStackConfig(object): # Flag location to hold the peeked value of an argparse timeout value self._argv_timeout = False + def get_extra_config(self, key, defaults=None): + """Fetch an arbitrary extra chunk of config, laying in defaults. + + :param string key: name of the config section to fetch + :param dict defaults: (optional) default values to merge under the + found config + """ + if not defaults: + defaults = {} + return _merge_clouds( + self._normalize_keys(defaults), + self._normalize_keys(self.cloud_config.get(key, {}))) + def _load_config_file(self): return self._load_yaml_json_file(self._config_files) diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 6d9e093d3..fdc50cd0b 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -121,6 +121,10 @@ USER_CONF = { 'region_name': 'test-region', } }, + 'ansible': { + 'expand-hostvars': False, + 'use_hostnames': True, + }, } SECURE_CONF = { 'clouds': { diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index a6a35ada9..98aaf79fc 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -372,6 +372,27 @@ class TestConfigArgparse(base.TestCase): self.assertDictEqual({'compute_api_version': 1}, fixed_args) + def test_extra_config(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + defaults = {'use_hostnames': False, 'other-value': 'something'} + ansible_options = c.get_extra_config('ansible', defaults) + + # This should show that the default for use_hostnames above is + # overridden by the value in the config file defined in base.py + # It should also show that other-value key is normalized and passed + # through even though there is no corresponding value in the config + # file, and that expand-hostvars key is normalized and the value + # from the config comes through even though there is no default. + self.assertDictEqual( + { + 'expand_hostvars': False, + 'use_hostnames': True, + 'other_value': 'something', + }, + ansible_options) + def test_register_argparse_cloud(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From 88b7e643b9637864252d9c6b715da0dced606352 Mon Sep 17 00:00:00 2001 From: Shuquan Huang Date: Thu, 17 Dec 2015 13:58:10 +0800 Subject: [PATCH 213/365] Replace assertEqual(None, *) with assertIsNone in tests Replace assertEqual(None, *) with assertIsNone in tests to have more clear messages in case of failure. Change-Id: Ia1af9f64f4f0a66c1429d81313b2c27a7c67cdd7 Closes-bug: #1280522 --- os_client_config/tests/test_cloud_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 9e683d1da..322973ad7 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -47,7 +47,7 @@ class TestCloudConfig(base.TestCase): self.assertEqual(1, cc.a) # Look up prefixed attribute, fail - returns None - self.assertEqual(None, cc.os_b) + self.assertIsNone(cc.os_b) # Look up straight value, then prefixed value self.assertEqual(3, cc.c) @@ -139,7 +139,7 @@ class TestCloudConfig(base.TestCase): self.assertEqual('region-al', cc.get_region_name()) self.assertEqual('region-al', cc.get_region_name('image')) self.assertEqual('region-bl', cc.get_region_name('compute')) - self.assertEqual(None, cc.get_api_version('image')) + self.assertIsNone(cc.get_api_version('image')) self.assertEqual('2', cc.get_api_version('compute')) self.assertEqual('mage', cc.get_service_type('image')) self.assertEqual('compute', cc.get_service_type('compute')) @@ -149,7 +149,7 @@ class TestCloudConfig(base.TestCase): cc.get_endpoint('compute')) self.assertEqual(None, cc.get_endpoint('image')) - self.assertEqual(None, cc.get_service_name('compute')) + self.assertIsNone(cc.get_service_name('compute')) self.assertEqual('locks', cc.get_service_name('identity')) def test_volume_override(self): From 22d740b7007e1182c99370cb2629322384b17a14 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 12 Dec 2015 10:53:53 -0500 Subject: [PATCH 214/365] Add backwards compat mapping for auth-token novaclient accepted an auth-token argument, which also triggered a token not password based workflow. That's fine - let's map that to token, and if we find it, change auth_type's default from password to token. Change-Id: Ie9acece5cb3c68560ae975bfb0fb2393381b6fba --- os_client_config/config.py | 15 ++++++++++++++ os_client_config/tests/test_config.py | 30 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index ab3a003c6..48bcb0f31 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -467,6 +467,7 @@ class OpenStackConfig(object): 'project_domain_id': ('project_domain_id', 'project-domain-id'), 'project_domain_name': ( 'project_domain_name', 'project-domain-name'), + 'token': ('auth-token', 'auth_token', 'token'), } for target_key, possible_values in mappings.items(): target = None @@ -535,6 +536,13 @@ class OpenStackConfig(object): # for from the user passing it explicitly. We'll stash it for later local_parser.add_argument('--timeout', metavar='') + # We need for get_one_cloud to be able to peek at whether a token + # was passed so that we can swap the default from password to + # token if it was. And we need to also peek for --os-auth-token + # for novaclient backwards compat + local_parser.add_argument('--os-token') + local_parser.add_argument('--os-auth-token') + # Peek into the future and see if we have an auth-type set in # config AND a cloud set, so that we know which command line # arguments to register and show to the user (the user may want @@ -832,6 +840,13 @@ class OpenStackConfig(object): else: config[key] = val + # Infer token plugin if a token was given + if (('auth' in config and 'token' in config['auth']) or + ('auth_token' in config and config['auth_token']) or + ('token' in config and config['token'])): + config['auth_type'] = 'token' + config.setdefault('token', config.pop('auth_token', None)) + # These backwards compat values are only set via argparse. If it's # there, it's because it was passed in explicitly, and should win config = self._fix_backwards_api_timeout(config) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 98aaf79fc..bb4569306 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -243,7 +243,9 @@ class TestConfigArgparse(base.TestCase): project_name='project', region_name='region2', snack_type='cookie', + os_auth_token='no-good-things', ) + self.options = argparse.Namespace(**self.args) def test_get_one_cloud_bad_region_argparse(self): @@ -401,6 +403,34 @@ class TestConfigArgparse(base.TestCase): opts, _remain = parser.parse_known_args(['--os-cloud', 'foo']) self.assertEqual(opts.os_cloud, 'foo') + def test_argparse_default_no_token(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + parser = argparse.ArgumentParser() + c.register_argparse_arguments(parser, []) + # novaclient will add this + parser.add_argument('--os-auth-token') + opts, _remain = parser.parse_known_args() + cc = c.get_one_cloud( + cloud='_test_cloud_regions', argparse=opts) + self.assertEqual(cc.config['auth_type'], 'password') + self.assertNotIn('token', cc.config['auth']) + + def test_argparse_token(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + parser = argparse.ArgumentParser() + c.register_argparse_arguments(parser, []) + # novaclient will add this + parser.add_argument('--os-auth-token') + opts, _remain = parser.parse_known_args( + ['--os-auth-token', 'very-bad-things']) + cc = c.get_one_cloud(argparse=opts) + self.assertEqual(cc.config['auth_type'], 'token') + self.assertEqual(cc.config['auth']['token'], 'very-bad-things') + def test_register_argparse_bad_plugin(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From 3a34378f712abee2d525973815a99188c598d726 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 12 Dec 2015 13:03:07 -0500 Subject: [PATCH 215/365] Support backwards compat for _ args Instead of putting tons of hidden options to allow for variations of argparse options with _ in them, just manipulate the argv when it's passed in to translate to - instead. (why the heck does argparse not already do this?) Change-Id: I5f0bd9d9a333781ad13d531b3667fff5fdac9eac --- os_client_config/config.py | 32 +++++++++++++++++++++++++++ os_client_config/tests/test_config.py | 31 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 48bcb0f31..077c109fe 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -15,6 +15,7 @@ # alias because we already had an option named argparse import argparse as argparse_mod +import collections import copy import json import os @@ -136,6 +137,34 @@ def _auth_update(old_dict, new_dict_source): return old_dict +def _fix_argv(argv): + # Transform any _ characters in arg names to - so that we don't + # have to throw billions of compat argparse arguments around all + # over the place. + processed = collections.defaultdict(list) + for index in range(0, len(argv)): + if argv[index].startswith('--'): + split_args = argv[index].split('=') + orig = split_args[0] + new = orig.replace('_', '-') + if orig != new: + split_args[0] = new + argv[index] = "=".join(split_args) + # Save both for later so we can throw an error about dupes + processed[new].append(orig) + overlap = [] + for new, old in processed.items(): + if len(old) > 1: + overlap.extend(old) + if overlap: + raise exceptions.OpenStackConfigException( + "The following options were given: '{options}' which contain" + " duplicates except that one has _ and one has -. There is" + " no sane way for us to know what you're doing. Remove the" + " duplicate option and try again".format( + options=','.join(overlap))) + + class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, @@ -521,6 +550,9 @@ class OpenStackConfig(object): is requested """ + # Fix argv in place - mapping any keys with embedded _ in them to - + _fix_argv(argv) + local_parser = argparse_mod.ArgumentParser(add_help=False) for p in (parser, local_parser): diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index bb4569306..30ed73188 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -431,6 +431,37 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(cc.config['auth_type'], 'token') self.assertEqual(cc.config['auth']['token'], 'very-bad-things') + def test_argparse_underscores(self): + c = config.OpenStackConfig(config_files=[self.no_yaml], + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) + parser = argparse.ArgumentParser() + parser.add_argument('--os_username') + argv = [ + '--os_username', 'user', '--os_password', 'pass', + '--os-auth-url', 'auth-url', '--os-project-name', 'project'] + c.register_argparse_arguments(parser, argv=argv) + opts, _remain = parser.parse_known_args(argv) + cc = c.get_one_cloud(argparse=opts) + self.assertEqual(cc.config['auth']['username'], 'user') + self.assertEqual(cc.config['auth']['password'], 'pass') + self.assertEqual(cc.config['auth']['auth_url'], 'auth-url') + + def test_argparse_underscores_duplicate(self): + c = config.OpenStackConfig(config_files=[self.no_yaml], + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) + parser = argparse.ArgumentParser() + parser.add_argument('--os_username') + argv = [ + '--os_username', 'user', '--os_password', 'pass', + '--os-username', 'user1', '--os-password', 'pass1', + '--os-auth-url', 'auth-url', '--os-project-name', 'project'] + self.assertRaises( + exceptions.OpenStackConfigException, + c.register_argparse_arguments, + parser=parser, argv=argv) + def test_register_argparse_bad_plugin(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From 16166c03c27fe73896a1717ad9a145a466bc0afd Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 12 Dec 2015 13:03:41 -0500 Subject: [PATCH 216/365] Pass endpoint override to constructors Also, the variable name from keystoneauth is "*-endpoint-override" ... so we need to respond to that. Respond to the old -endpoint for compat reasons. Then let's actually pass in the value. Change-Id: I6f413b02e0d2b167a4ee30494b2c91c67124b219 --- os_client_config/cloud_config.py | 6 ++++-- os_client_config/tests/test_cloud_config.py | 22 ++++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 6b3b5d9a0..2f3c94ab2 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -164,8 +164,9 @@ class CloudConfig(object): return self.config.get(key, None) def get_endpoint(self, service_type): - key = _make_key('endpoint', service_type) - return self.config.get(key, None) + key = _make_key('endpoint_override', service_type) + old_key = _make_key('endpoint', service_type) + return self.config.get(key, self.config.get(old_key, None)) @property def prefer_ipv6(self): @@ -310,6 +311,7 @@ class CloudConfig(object): session=self.get_session(), service_name=self.get_service_name(service_key), service_type=self.get_service_type(service_key), + endpoint_override=self.get_endpoint(service_key), region_name=self.region) if service_key == 'image': diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 322973ad7..341225482 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -25,8 +25,9 @@ from os_client_config.tests import base fake_config_dict = {'a': 1, 'os_b': 2, 'c': 3, 'os_c': 4} fake_services_dict = { 'compute_api_version': '2', - 'compute_endpoint': 'http://compute.example.com', + 'compute_endpoint_override': 'http://compute.example.com', 'compute_region_name': 'region-bl', + 'telemetry_endpoint': 'http://telemetry.example.com', 'interface': 'public', 'image_service_type': 'mage', 'identity_interface': 'admin', @@ -189,14 +190,24 @@ class TestCloudConfig(base.TestCase): verify=True, cert=None, timeout=9) @mock.patch.object(ksa_session, 'Session') - def test_override_session_endpoint(self, mock_session): + def test_override_session_endpoint_override(self, mock_session): config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) cc = cloud_config.CloudConfig( "test1", "region-al", config_dict, auth_plugin=mock.Mock()) self.assertEqual( cc.get_session_endpoint('compute'), - fake_services_dict['compute_endpoint']) + fake_services_dict['compute_endpoint_override']) + + @mock.patch.object(ksa_session, 'Session') + def test_override_session_endpoint(self, mock_session): + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + self.assertEqual( + cc.get_session_endpoint('telemetry'), + fake_services_dict['telemetry_endpoint']) @mock.patch.object(cloud_config.CloudConfig, 'get_session') def test_session_endpoint_identity(self, mock_get_session): @@ -297,6 +308,7 @@ class TestCloudConfig(base.TestCase): '2', service_name=None, endpoint='http://example.com', + endpoint_override=None, region_name='region-al', interface='public', session=mock.ANY, @@ -316,6 +328,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( '2.0', endpoint_type='public', + endpoint_override=None, region_name='region-al', service_type='network', session=mock.ANY, @@ -333,6 +346,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( '2', endpoint_type='public', + endpoint_override='http://compute.example.com', region_name='region-al', service_type='compute', session=mock.ANY, @@ -351,6 +365,7 @@ class TestCloudConfig(base.TestCase): '2.0', endpoint='http://example.com/v2', endpoint_type='admin', + endpoint_override=None, region_name='region-al', service_type='identity', session=mock.ANY, @@ -370,6 +385,7 @@ class TestCloudConfig(base.TestCase): '3', endpoint='http://example.com', endpoint_type='admin', + endpoint_override=None, region_name='region-al', service_type='identity', session=mock.ANY, From 0a25cb5c5059fc8f850e27a984613515ae76ee11 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 12 Dec 2015 17:26:09 -0500 Subject: [PATCH 217/365] Allow passing in explicit version for legacy_client Nova (and indeed other clients with microversions, need a user to be able to request an explicit version. Change-Id: I5f67b7fc007b7d6123f621c5943345f88db1f84b --- os_client_config/cloud_config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 2f3c94ab2..0233eff81 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -255,7 +255,7 @@ class CloudConfig(object): def get_legacy_client( self, service_key, client_class=None, interface_key=None, - pass_version_arg=True, **kwargs): + pass_version_arg=True, version=None, **kwargs): """Return a legacy OpenStack client object for the given config. Most of the OpenStack python-*client libraries have the same @@ -287,6 +287,8 @@ class CloudConfig(object): already understand that this is the case for network, so it can be omitted in that case. + :param version: (optional) Version string to override the configured + version string. :param kwargs: (optional) keyword args are passed through to the Client constructor, so this is in case anything additional needs to be passed in. @@ -320,13 +322,14 @@ class CloudConfig(object): # would need to do if they were requesting 'image' - then # they necessarily have glanceclient installed from glanceclient.common import utils as glance_utils - endpoint, version = glance_utils.strip_version(endpoint) + endpoint, _ = glance_utils.strip_version(endpoint) constructor_kwargs['endpoint'] = endpoint constructor_kwargs.update(kwargs) constructor_kwargs[interface_key] = interface constructor_args = [] if pass_version_arg: - version = self.get_api_version(service_key) + if not version: + version = self.get_api_version(service_key) # Temporary workaround while we wait for python-openstackclient # to be able to handle 2.0 which is what neutronclient expects if service_key == 'network' and version == '2': From 939862e55e42c5fafee9c2fec42b5f5fde8fc205 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 21 Dec 2015 11:35:56 -0600 Subject: [PATCH 218/365] Fix glance endpoints with endpoint_override Now that we properly pass endpoint_override all the time, we broke glance. The reason for this is that we calculate the glance url via glance url stripping in all cases, so the case where we did not have a configured endpoint override was passing the wrong information to the constructor, causing double version addition. Change-Id: I5699b0581d0cb68fed68800c29c8a847e2606ec9 --- os_client_config/cloud_config.py | 15 +++- os_client_config/tests/test_cloud_config.py | 92 ++++++++++++++++++++- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 0233eff81..f73da04ed 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -302,6 +302,7 @@ class CloudConfig(object): interface = self.get_interface(service_key) # trigger exception on lack of service endpoint = self.get_session_endpoint(service_key) + endpoint_override = self.get_endpoint(service_key) if not interface_key: if service_key == 'image': @@ -313,7 +314,7 @@ class CloudConfig(object): session=self.get_session(), service_name=self.get_service_name(service_key), service_type=self.get_service_type(service_key), - endpoint_override=self.get_endpoint(service_key), + endpoint_override=endpoint_override, region_name=self.region) if service_key == 'image': @@ -322,8 +323,16 @@ class CloudConfig(object): # would need to do if they were requesting 'image' - then # they necessarily have glanceclient installed from glanceclient.common import utils as glance_utils - endpoint, _ = glance_utils.strip_version(endpoint) - constructor_kwargs['endpoint'] = endpoint + endpoint, detected_version = glance_utils.strip_version(endpoint) + # If the user has passed in a version, that's explicit, use it + if not version: + version = detected_version + # If the user has passed in or configured an override, use it. + # Otherwise, ALWAYS pass in an endpoint_override becuase + # we've already done version stripping, so we don't want version + # reconstruction to happen twice + if not endpoint_override: + constructor_kwargs['endpoint_override'] = endpoint constructor_kwargs.update(kwargs) constructor_kwargs[interface_key] = interface constructor_args = [] diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 341225482..01581b1fb 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -305,10 +305,96 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('image', mock_client) mock_client.assert_called_with( - '2', + 2.0, service_name=None, - endpoint='http://example.com', - endpoint_override=None, + endpoint_override='http://example.com', + region_name='region-al', + interface='public', + session=mock.ANY, + # Not a typo - the config dict above overrides this + service_type='mage' + ) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_image_override(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + config_dict['image_endpoint_override'] = 'http://example.com/override' + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('image', mock_client) + mock_client.assert_called_with( + 2.0, + service_name=None, + endpoint_override='http://example.com/override', + region_name='region-al', + interface='public', + session=mock.ANY, + # Not a typo - the config dict above overrides this + service_type='mage' + ) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_image_versioned(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + # v2 endpoint was passed, 1 requested in config, endpoint wins + config_dict['image_api_version'] = '1' + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('image', mock_client) + mock_client.assert_called_with( + 2.0, + service_name=None, + endpoint_override='http://example.com', + region_name='region-al', + interface='public', + session=mock.ANY, + # Not a typo - the config dict above overrides this + service_type='mage' + ) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_image_unversioned(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + # Versionless endpoint, config wins + config_dict['image_api_version'] = '1' + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('image', mock_client) + mock_client.assert_called_with( + '1', + service_name=None, + endpoint_override='http://example.com', + region_name='region-al', + interface='public', + session=mock.ANY, + # Not a typo - the config dict above overrides this + service_type='mage' + ) + + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_image_argument(self, mock_get_session_endpoint): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v3' + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + # Versionless endpoint, config wins + config_dict['image_api_version'] = '6' + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('image', mock_client, version='beef') + mock_client.assert_called_with( + 'beef', + service_name=None, + endpoint_override='http://example.com', region_name='region-al', interface='public', session=mock.ANY, From 8fa55700b90e335e54cd459ea8a60578e8d27fc7 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 22 Dec 2015 12:28:20 -0800 Subject: [PATCH 219/365] If cloud doesn't list regions expand passed name Don't fail on a cloud not having regions when a region name is passed. Instead just use the name that is given and expand it properly. This adds test coverage for the paths through the OpenStackConfig._get_region() method to avoid problems like this in the future. In order for this work to be done cleanly a small refactor of get_regions() is done to split it into two methods, one that gets all regions with a sane fallback default (for backward compat) and another that returns only regions that are known in the config and None otherwise. This allows us to switch on whether or not there are known regions. Change-Id: I62736ea82f365badaea5016a23d37a9f1c760927 --- os_client_config/config.py | 15 +++++-- os_client_config/tests/base.py | 10 ++++- os_client_config/tests/test_config.py | 56 +++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 077c109fe..89015cc90 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -366,6 +366,13 @@ class OpenStackConfig(object): def _get_regions(self, cloud): if cloud not in self.cloud_config['clouds']: return [self._expand_region_name('')] + regions = self._get_known_regions(cloud) + if not regions: + # We don't know of any regions use a workable default. + regions = [self._expand_region_name('')] + return regions + + def _get_known_regions(self, cloud): config = self._normalize_keys(self.cloud_config['clouds'][cloud]) if 'regions' in config: return self._expand_regions(config['regions']) @@ -386,15 +393,15 @@ class OpenStackConfig(object): return self._expand_regions(new_cloud['regions']) elif 'region_name' in new_cloud and new_cloud['region_name']: return [self._expand_region_name(new_cloud['region_name'])] - else: - # Wow. We really tried - return [self._expand_region_name('')] def _get_region(self, cloud=None, region_name=''): if not cloud: return self._expand_region_name(region_name) - regions = self._get_regions(cloud) + regions = self._get_known_regions(cloud) + if not regions: + return self._expand_region_name(region_name) + if not region_name: return regions[0] diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index fdc50cd0b..3f00f6d82 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -119,7 +119,15 @@ USER_CONF = { 'auth_url': 'http://example.com/v2', }, 'region_name': 'test-region', - } + }, + '_test-cloud_no_region': { + 'profile': '_test_cloud_in_our_cloud', + 'auth': { + 'auth_url': 'http://example.com/v2', + 'username': 'testuser', + 'password': 'testpass', + }, + }, }, 'ansible': { 'expand-hostvars': False, diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 30ed73188..3ea6690e2 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -178,6 +178,7 @@ class TestConfig(base.TestCase): ['_test-cloud-domain-id_', '_test-cloud-int-project_', '_test-cloud_', + '_test-cloud_no_region', '_test_cloud_hyphenated', '_test_cloud_no_vendor', '_test_cloud_regions', @@ -230,6 +231,61 @@ class TestConfig(base.TestCase): written_config['cache'].pop('path', None) self.assertEqual(written_config, resulting_config) + def test_get_region_no_region_default(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) + region = c._get_region(cloud='_test-cloud_no_region') + self.assertEqual(region, {'name': '', 'values': {}}) + + def test_get_region_no_region(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) + region = c._get_region(cloud='_test-cloud_no_region', + region_name='override-region') + self.assertEqual(region, {'name': 'override-region', 'values': {}}) + + def test_get_region_region_set(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) + region = c._get_region(cloud='_test-cloud_', region_name='test-region') + self.assertEqual(region, {'name': 'test-region', 'values': {}}) + + def test_get_region_many_regions_default(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) + region = c._get_region(cloud='_test_cloud_regions', + region_name='') + self.assertEqual(region, {'name': 'region1', 'values': + {'external_network': 'region1-network'}}) + + def test_get_region_many_regions(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) + region = c._get_region(cloud='_test_cloud_regions', + region_name='region2') + self.assertEqual(region, {'name': 'region2', 'values': + {'external_network': 'my-network'}}) + + def test_get_region_invalid_region(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) + self.assertRaises( + exceptions.OpenStackConfigException, c._get_region, + cloud='_test_cloud_regions', region_name='invalid-region') + + def test_get_region_no_cloud(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) + region = c._get_region(region_name='no-cloud-region') + self.assertEqual(region, {'name': 'no-cloud-region', 'values': {}}) + class TestConfigArgparse(base.TestCase): From f765a16c6c84d8b4a116585676f46087e458987b Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 23 Dec 2015 01:31:13 +0000 Subject: [PATCH 220/365] remove python 2.6 os-client-config classifier OpenStack projects are no longer being tested under Python 2.6, so remove the classifier implying that this project supports 2.6. Change-Id: Ic24f93d5f7e7ffb1eaf91617c09cc897163e88df --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index bc4f128cf..89df35c11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,6 @@ classifier = Programming Language :: Python Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 - Programming Language :: Python :: 2.6 Programming Language :: Python :: 3 Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 From 77c0365ce2d42adce9352cf238fbdbc7c282222f Mon Sep 17 00:00:00 2001 From: Javier Pena Date: Wed, 23 Dec 2015 11:38:40 +0100 Subject: [PATCH 221/365] Fix token_endpoint usage Commit 22d740b7007e1182c99370cb2629322384b17a14 broke token_endpoint authentication for openstackclient, by unconditionally setting auth_type to 'token' whenever a token was passed in the command line. This change reverts the portion that always overrides the auth plugin if there is a token passed via arguments. Change-Id: I835c3716dd08eaca10f56682c22fdc6ac700e0fe --- os_client_config/config.py | 1 - os_client_config/tests/test_config.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 89015cc90..51019837e 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -883,7 +883,6 @@ class OpenStackConfig(object): if (('auth' in config and 'token' in config['auth']) or ('auth_token' in config and config['auth_token']) or ('token' in config and config['token'])): - config['auth_type'] = 'token' config.setdefault('token', config.pop('auth_token', None)) # These backwards compat values are only set via argparse. If it's diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 3ea6690e2..aef97371a 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -482,7 +482,8 @@ class TestConfigArgparse(base.TestCase): # novaclient will add this parser.add_argument('--os-auth-token') opts, _remain = parser.parse_known_args( - ['--os-auth-token', 'very-bad-things']) + ['--os-auth-token', 'very-bad-things', + '--os-auth-type', 'token']) cc = c.get_one_cloud(argparse=opts) self.assertEqual(cc.config['auth_type'], 'token') self.assertEqual(cc.config['auth']['token'], 'very-bad-things') From 7be6db82d35a7c3402172e742bb19dfa5a95a472 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 28 Dec 2015 16:43:26 -0800 Subject: [PATCH 222/365] Fix some README typos Change-Id: I3ebec661d1b02da0c940cde63ab862871dca11c5 --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index ced3b1821..811c2d3e6 100644 --- a/README.rst +++ b/README.rst @@ -117,7 +117,7 @@ An example config file is probably helpful: - IAD You may note a few things. First, since `auth_url` settings are silly -and embarrasingly ugly, known cloud vendor profile information is included and +and embarrassingly ugly, known cloud vendor profile information is included and may be referenced by name. One of the benefits of that is that `auth_url` isn't the only thing the vendor defaults contain. For instance, since Rackspace lists `rax:database` as the service type for trove, `os-client-config` @@ -148,8 +148,8 @@ related to domains, projects and trusts. Splitting Secrets ----------------- -In some scenarios, such as configuragtion managment controlled environments, -it might be eaiser to have secrets in one file and non-secrets in another. +In some scenarios, such as configuration management controlled environments, +it might be easier to have secrets in one file and non-secrets in another. This is fully supported via an optional file `secure.yaml` which follows all the same location rules as `clouds.yaml`. It can contain anything you put in `clouds.yaml` and will take precedence over anything in the `clouds.yaml` @@ -380,4 +380,4 @@ If you want to do the same thing but also support command line parsing. 'compute', options=argparse.ArgumentParser()) If you want to get fancier than that in your python, then the rest of the -API is avaiable to you. But often times, you just want to do the one thing. +API is available to you. But often times, you just want to do the one thing. From 17e019a08e6e8fed7da6d0de403e5525d997095b Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Tue, 29 Dec 2015 15:22:56 -0800 Subject: [PATCH 223/365] Munge region_name to '' if set to None The openstack ansible module defaults to setting region_name to None[1]. With region_name explicitly set, _get_region won't use '' as a default and therefor has unexpected behavior if the user does not set the region explicitly. This is apparent in bifrost[2] which does not use any cloud config file and does not set the region explicitly. This patch checks whether None was passed in as the region name and sets it to '' so that it can continue processing it as though it was not set. [1] https://github.com/ansible/ansible/blob/devel/lib/ansible/module_utils/openstack.py#L41 [2] http://paste.openstack.org/show/482831/ Change-Id: I22cce104930f74dd479e704cc1a941dc945b75de --- os_client_config/config.py | 2 ++ os_client_config/tests/test_config.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 89015cc90..d5b1ab552 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -395,6 +395,8 @@ class OpenStackConfig(object): return [self._expand_region_name(new_cloud['region_name'])] def _get_region(self, cloud=None, region_name=''): + if region_name is None: + region_name = '' if not cloud: return self._expand_region_name(region_name) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 3ea6690e2..b2ee9bba8 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -246,6 +246,13 @@ class TestConfig(base.TestCase): region_name='override-region') self.assertEqual(region, {'name': 'override-region', 'values': {}}) + def test_get_region_region_is_none(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) + region = c._get_region(cloud='_test-cloud_no_region', region_name=None) + self.assertEqual(region, {'name': '', 'values': {}}) + def test_get_region_region_set(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml], From 7ee7156254381dc5c06405105c7de42c180c779f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 30 Dec 2015 09:46:21 -0600 Subject: [PATCH 224/365] Allow filtering clouds on command line Add a very basic filtering to the test command line function to allow only printing one cloud or one cloud/region worth of config. Change-Id: I0d09717430f41b4229f7743f8531f871b962969e --- os_client_config/config.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 077c109fe..b970728e0 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -19,6 +19,7 @@ import collections import copy import json import os +import sys import warnings import appdirs @@ -976,4 +977,15 @@ class OpenStackConfig(object): if __name__ == '__main__': config = OpenStackConfig().get_all_clouds() for cloud in config: - print(cloud.name, cloud.region, cloud.config) + print_cloud = False + if len(sys.argv) == 1: + print_cloud = True + elif len(sys.argv) == 3 and ( + sys.argv[1] == cloud.name and sys.argv[2] == cloud.region): + print_cloud = True + elif len(sys.argv) == 2 and ( + sys.argv[1] == cloud.name): + print_cloud = True + + if print_cloud: + print(cloud.name, cloud.region, cloud.config) From f3678f03deac0230e1265a8a516a8eea11d301cf Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 30 Dec 2015 19:06:38 +0000 Subject: [PATCH 225/365] add URLs for release announcement tools The release announcement scripts expects to find URLs for the bug tracker, documentation, etc. by looking for patterns in the README.rst file. This change adds the URLs in a format consistent with other OpenStack projects and that works with the release announcement generator. Change-Id: I88151008cca91da3fed7e4c0ec6dfb641a0062b6 --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index 811c2d3e6..5cbc118e5 100644 --- a/README.rst +++ b/README.rst @@ -381,3 +381,11 @@ If you want to do the same thing but also support command line parsing. If you want to get fancier than that in your python, then the rest of the API is available to you. But often times, you just want to do the one thing. + +Source +------ + +* Free software: Apache license +* Documentation: http://docs.openstack.org/developer/os-client-config +* Source: http://git.openstack.org/cgit/openstack/os-client-config +* Bugs: http://bugs.launchpad.net/os-client-config From 594e31a4c262c9ae3fe14e2e4c0fdb71a0df0747 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 30 Dec 2015 13:10:47 -0600 Subject: [PATCH 226/365] Use reno for release notes The OpenStack Release team has created a great release notes management tool that integrates with Sphinx. Start using it. For reference on how to use it, see http://docs.openstack.org/developer/reno/ Change-Id: I8153ec7861b508297a28a1916771776dee2deafe --- doc/source/conf.py | 3 ++- doc/source/index.rst | 1 + doc/source/releasenotes.rst | 5 +++++ releasenotes/notes/started-using-reno-242e2b0cd27f9480.yaml | 3 +++ test-requirements.txt | 1 + 5 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 doc/source/releasenotes.rst create mode 100644 releasenotes/notes/started-using-reno-242e2b0cd27f9480.yaml diff --git a/doc/source/conf.py b/doc/source/conf.py index 221de3c88..208517c86 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,7 +23,8 @@ sys.path.insert(0, os.path.abspath('../..')) extensions = [ 'sphinx.ext.autodoc', #'sphinx.ext.intersphinx', - 'oslosphinx' + 'oslosphinx', + 'reno.sphinxext' ] # autodoc generation is a bit aggressive and a nuisance when doing heavy diff --git a/doc/source/index.rst b/doc/source/index.rst index cc5dbf470..bf667b786 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,6 +7,7 @@ contributing installation api-reference + releasenotes Indices and tables ================== diff --git a/doc/source/releasenotes.rst b/doc/source/releasenotes.rst new file mode 100644 index 000000000..2a4bceb4e --- /dev/null +++ b/doc/source/releasenotes.rst @@ -0,0 +1,5 @@ +============= +Release Notes +============= + +.. release-notes:: diff --git a/releasenotes/notes/started-using-reno-242e2b0cd27f9480.yaml b/releasenotes/notes/started-using-reno-242e2b0cd27f9480.yaml new file mode 100644 index 000000000..d7cfb5145 --- /dev/null +++ b/releasenotes/notes/started-using-reno-242e2b0cd27f9480.yaml @@ -0,0 +1,3 @@ +--- +other: +- Started using reno for release notes. diff --git a/test-requirements.txt b/test-requirements.txt index 70530517e..a50a202e3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -16,6 +16,7 @@ python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 oslosphinx>=2.5.0,<2.6.0 # Apache-2.0 oslotest>=1.5.1,<1.6.0 # Apache-2.0 +reno>=0.1.1 # Apache2 testrepository>=0.0.18 testscenarios>=0.4 testtools>=0.9.36,!=1.2.0 From 9688f8ebd1ace0f338a1eabb77e1bee249e5630b Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Thu, 31 Dec 2015 15:40:58 +0300 Subject: [PATCH 227/365] Fix README.rst, add a check for it to fit PyPI rules README.rst doesn't appear right on PyPI currently. This commit fixes the issue and expands "docs" environment in tox.ini to use readme tool [0] to verify that README.rst is good for PyPI. [0] https://github.com/pypa/readme Change-Id: I6025bb6c661d8a4a7cd9802a1298928662278f2d --- README.rst | 2 +- tox.ini | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5cbc118e5..f078f3cee 100644 --- a/README.rst +++ b/README.rst @@ -355,7 +355,7 @@ with - as well as a consumption argument. Constructing Legacy Client objects ---------------------------------- -If all you want to do is get a Client object from a python-*client library, +If all you want to do is get a Client object from a python-\*client library, and you want it to do all the normal things related to clouds.yaml, `OS_` environment variables, a helper function is provided. The following will get you a fully configured `novaclient` instance. diff --git a/tox.ini b/tox.ini index 7a2d3a07a..95dff6ba4 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,12 @@ commands = {posargs} commands = python setup.py test --coverage --coverage-package-name=os_client_config --testr-args='{posargs}' [testenv:docs] -commands = python setup.py build_sphinx +deps = + {[testenv]deps} + readme +commands = + python setup.py build_sphinx + python setup.py check -r -s [flake8] # H803 skipped on purpose per list discussion. From c514b855d1faed8947ace885bb4656da541d4d2b Mon Sep 17 00:00:00 2001 From: Doug Wiegley Date: Thu, 31 Dec 2015 12:32:37 -0700 Subject: [PATCH 228/365] Debug log a deferred keystone exception, else we mask some useful diag Change-Id: Ib1921698bb61f44193034065749b4e246a6258db --- os_client_config/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index b57264572..d490006b4 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -26,6 +26,7 @@ from keystoneauth1 import adapter from keystoneauth1 import loading import yaml +from os_client_config import _log from os_client_config import cloud_config from os_client_config import defaults from os_client_config import exceptions @@ -170,6 +171,8 @@ class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, override_defaults=None, force_ipv4=None, envvar_prefix=None, secure_files=None): + self.log = _log.setup_logging(__name__) + self._config_files = config_files or CONFIG_FILES self._secure_files = secure_files or SECURE_FILES self._vendor_files = vendor_files or VENDOR_FILES @@ -920,6 +923,7 @@ class OpenStackConfig(object): # but OSC can't handle it right now, so we try deferring # to ksc. If that ALSO fails, it means there is likely # a deeper issue, so we assume the ksa error was correct + self.log.debug("Deferring keystone exception: {e}".format(e=e)) auth_plugin = None try: config = self._validate_auth_ksc(config) From 1cd3e5bb7fd7cd72a481f5ae8bbcd0b2ab114680 Mon Sep 17 00:00:00 2001 From: Yaguang Tang Date: Sun, 27 Dec 2015 10:59:08 +0800 Subject: [PATCH 229/365] Update volume API default version from v1 to v2 Cinder has deprecated API version v1 since Juno release, and there is a blueprint to remove v1 API support which is in progress. We should default to v2 API when it's there. Closes-Bug: 1467589 Change-Id: I83aef4c681cbe342c445f02436fcd40cf1222f23 --- doc/source/vendor-support.rst | 11 +++++++++++ os_client_config/defaults.json | 2 +- os_client_config/vendors/bluebox.json | 1 + os_client_config/vendors/catalyst.json | 1 + os_client_config/vendors/citycloud.json | 1 + os_client_config/vendors/entercloudsuite.json | 1 + os_client_config/vendors/hp.json | 1 + os_client_config/vendors/rackspace.json | 1 + os_client_config/vendors/switchengines.json | 1 + os_client_config/vendors/ultimum.json | 1 + os_client_config/vendors/unitedstack.json | 1 + 11 files changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index d7af6b913..8ae2f6194 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -16,6 +16,7 @@ These are the default behaviors unless a cloud is configured differently. * Identity uses `password` authentication * Identity API Version is 2 * Image API Version is 2 +* Volume API Version is 2 * Images must be in `qcow2` format * Images are uploaded using PUT interface * Public IPv4 is directly routable via DHCP from Neutron @@ -51,6 +52,7 @@ nz_wlg_2 Wellington, NZ * Image API Version is 1 * Images must be in `raw` format +* Volume API Version is 1 citycloud --------- @@ -67,6 +69,7 @@ Kna1 Karlskrona, SE * Identity API Version is 3 * Public IPv4 is provided via NAT with Neutron Floating IP +* Volume API Version is 1 conoha ------ @@ -137,6 +140,8 @@ it-mil1 Milan, IT de-fra1 Frankfurt, DE ============== ================ +* Volume API Version is 1 + hp -- @@ -152,6 +157,7 @@ region-b.geo-1 US East * DNS Service Type is `hpext:dns` * Image API Version is 1 * Public IPv4 is provided via NAT with Neutron Floating IP +* Volume API Version is 1 internap -------- @@ -212,6 +218,7 @@ SYD Sydney * Uploaded Images need properties to not use vendor agent:: :vm_mode: hvm :xenapi_use_agent: False +* Volume API Version is 1 runabove -------- @@ -241,6 +248,7 @@ ZH Zurich, CH * Images must be in `raw` format * Images must be uploaded using the Glance Task Interface +* Volume API Version is 1 ultimum ------- @@ -253,6 +261,8 @@ Region Name Human Name RegionOne Region One ============== ================ +* Volume API Version is 1 + unitedstack ----------- @@ -267,6 +277,7 @@ gd1 Guangdong * Identity API Version is 3 * Images must be in `raw` format +* Volume API Version is 1 vexxhost -------- diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index 6735b5535..2ffb67262 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -17,5 +17,5 @@ "object_store_api_version": "1", "orchestration_api_version": "1", "secgroup_source": "neutron", - "volume_api_version": "1" + "volume_api_version": "2" } diff --git a/os_client_config/vendors/bluebox.json b/os_client_config/vendors/bluebox.json index 2227aac81..647c8429f 100644 --- a/os_client_config/vendors/bluebox.json +++ b/os_client_config/vendors/bluebox.json @@ -1,6 +1,7 @@ { "name": "bluebox", "profile": { + "volume_api_version": "1", "region_name": "RegionOne" } } diff --git a/os_client_config/vendors/catalyst.json b/os_client_config/vendors/catalyst.json index ddde83825..3ad75075b 100644 --- a/os_client_config/vendors/catalyst.json +++ b/os_client_config/vendors/catalyst.json @@ -9,6 +9,7 @@ "nz_wlg_2" ], "image_api_version": "1", + "volume_api_version": "1", "image_format": "raw" } } diff --git a/os_client_config/vendors/citycloud.json b/os_client_config/vendors/citycloud.json index f6c57c7b5..64cadce9a 100644 --- a/os_client_config/vendors/citycloud.json +++ b/os_client_config/vendors/citycloud.json @@ -9,6 +9,7 @@ "Sto2", "Kna1" ], + "volume_api_version": "1", "identity_api_version": "3" } } diff --git a/os_client_config/vendors/entercloudsuite.json b/os_client_config/vendors/entercloudsuite.json index 826c25f7f..5a425b4c1 100644 --- a/os_client_config/vendors/entercloudsuite.json +++ b/os_client_config/vendors/entercloudsuite.json @@ -4,6 +4,7 @@ "auth": { "auth_url": "https://api.entercloudsuite.com/v2.0" }, + "volume_api_version": "1", "regions": [ "it-mil1", "nl-ams1", diff --git a/os_client_config/vendors/hp.json b/os_client_config/vendors/hp.json index 10789a91b..ac280f2d1 100644 --- a/os_client_config/vendors/hp.json +++ b/os_client_config/vendors/hp.json @@ -9,6 +9,7 @@ "region-b.geo-1" ], "dns_service_type": "hpext:dns", + "volume_api_version": "1", "image_api_version": "1" } } diff --git a/os_client_config/vendors/rackspace.json b/os_client_config/vendors/rackspace.json index 582e1225c..3fbbacd90 100644 --- a/os_client_config/vendors/rackspace.json +++ b/os_client_config/vendors/rackspace.json @@ -18,6 +18,7 @@ "image_format": "vhd", "floating_ip_source": "None", "secgroup_source": "None", + "volume_api_version": "1", "disable_vendor_agent": { "vm_mode": "hvm", "xenapi_use_agent": false diff --git a/os_client_config/vendors/switchengines.json b/os_client_config/vendors/switchengines.json index 8a7c566b8..46f632515 100644 --- a/os_client_config/vendors/switchengines.json +++ b/os_client_config/vendors/switchengines.json @@ -8,6 +8,7 @@ "LS", "ZH" ], + "volume_api_version": "1", "image_api_use_tasks": true, "image_format": "raw" } diff --git a/os_client_config/vendors/ultimum.json b/os_client_config/vendors/ultimum.json index ada6e3de7..0b38d71db 100644 --- a/os_client_config/vendors/ultimum.json +++ b/os_client_config/vendors/ultimum.json @@ -4,6 +4,7 @@ "auth": { "auth_url": "https://console.ultimum-cloud.com:5000/v2.0" }, + "volume_api_version": "1", "region-name": "RegionOne" } } diff --git a/os_client_config/vendors/unitedstack.json b/os_client_config/vendors/unitedstack.json index 41f45851a..ac8be117f 100644 --- a/os_client_config/vendors/unitedstack.json +++ b/os_client_config/vendors/unitedstack.json @@ -8,6 +8,7 @@ "bj1", "gd1" ], + "volume_api_version": "1", "identity_api_version": "3", "image_format": "raw", "floating_ip_source": "None" From 0bc9e33c9f978a8262453d7364143e8a02d3eded Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 6 Jan 2016 08:43:19 -0600 Subject: [PATCH 230/365] Stop hardcoding compute in simple_client There's a debug leftover oops where we just passed 'compute' rather than the service_key requested. Change-Id: Id8c82e43ba34859426b1fdc93dcf3ab2bbde4966 --- os_client_config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index ac585f248..ece155983 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -31,7 +31,7 @@ def simple_client(service_key, cloud=None, region_name=None): at OpenStack REST APIs with a properly configured keystone session. """ return OpenStackConfig().get_one_cloud( - cloud=cloud, region_name=region_name).get_session_client('compute') + cloud=cloud, region_name=region_name).get_session_client(service_key) def make_client(service_key, constructor, options=None, **kwargs): From 3b5673ce4c8c9d54568028056300eae053828ee0 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 5 Jan 2016 10:30:28 -0600 Subject: [PATCH 231/365] Update auth urls and identity API versions Most of the clouds, it turns out, support unversioned auth_url as well as keystone v3. Change-Id: I088d008cd2732f137c8a1bbbd9c0a43f7d382f92 --- doc/source/vendor-support.rst | 4 ++-- os_client_config/vendors/auro.json | 1 + os_client_config/vendors/conoha.json | 5 +++-- os_client_config/vendors/datacentred.json | 3 ++- os_client_config/vendors/dreamhost.json | 3 ++- os_client_config/vendors/elastx.json | 3 ++- os_client_config/vendors/entercloudsuite.json | 3 ++- os_client_config/vendors/hp.json | 3 ++- os_client_config/vendors/internap.json | 3 ++- os_client_config/vendors/ovh.json | 3 ++- os_client_config/vendors/runabove.json | 3 ++- os_client_config/vendors/ultimum.json | 3 ++- 12 files changed, 24 insertions(+), 13 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 8ae2f6194..a215822da 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -74,7 +74,7 @@ Kna1 Karlskrona, SE conoha ------ -https://identity.%(region_name)s.conoha.io/v2.0 +https://identity.%(region_name)s.conoha.io ============== ================ Region Name Human Name @@ -89,7 +89,7 @@ sjc1 San Jose, CA datacentred ----------- -https://compute.datacentred.io:5000/v2.0 +https://compute.datacentred.io:5000 ============== ================ Region Name Human Name diff --git a/os_client_config/vendors/auro.json b/os_client_config/vendors/auro.json index 1e59f01d4..a9e709bea 100644 --- a/os_client_config/vendors/auro.json +++ b/os_client_config/vendors/auro.json @@ -4,6 +4,7 @@ "auth": { "auth_url": "https://api.van1.auro.io:5000/v2.0" }, + "identity_api_version": "2", "region_name": "van1" } } diff --git a/os_client_config/vendors/conoha.json b/os_client_config/vendors/conoha.json index 8e33ca41b..5636f0955 100644 --- a/os_client_config/vendors/conoha.json +++ b/os_client_config/vendors/conoha.json @@ -2,12 +2,13 @@ "name": "conoha", "profile": { "auth": { - "auth_url": "https://identity.{region_name}.conoha.io/v2.0" + "auth_url": "https://identity.{region_name}.conoha.io" }, "regions": [ "sin1", "sjc1", "tyo1" - ] + ], + "identity_api_version": "2" } } diff --git a/os_client_config/vendors/datacentred.json b/os_client_config/vendors/datacentred.json index 1fb4dbb09..2be4a5863 100644 --- a/os_client_config/vendors/datacentred.json +++ b/os_client_config/vendors/datacentred.json @@ -2,9 +2,10 @@ "name": "datacentred", "profile": { "auth": { - "auth_url": "https://compute.datacentred.io:5000/v2.0" + "auth_url": "https://compute.datacentred.io:5000" }, "region-name": "sal01", + "identity_api_version": "2", "image_api_version": "1" } } diff --git a/os_client_config/vendors/dreamhost.json b/os_client_config/vendors/dreamhost.json index 8580826e1..6fc2ccf8a 100644 --- a/os_client_config/vendors/dreamhost.json +++ b/os_client_config/vendors/dreamhost.json @@ -2,8 +2,9 @@ "name": "dreamhost", "profile": { "auth": { - "auth_url": "https://keystone.dream.io/v2.0" + "auth_url": "https://keystone.dream.io" }, + "identity_api_version": "3", "region_name": "RegionOne", "image_format": "raw" } diff --git a/os_client_config/vendors/elastx.json b/os_client_config/vendors/elastx.json index cac755e8f..1e7248213 100644 --- a/os_client_config/vendors/elastx.json +++ b/os_client_config/vendors/elastx.json @@ -2,8 +2,9 @@ "name": "elastx", "profile": { "auth": { - "auth_url": "https://ops.elastx.net:5000/v2.0" + "auth_url": "https://ops.elastx.net:5000" }, + "identity_api_version": "3", "region_name": "regionOne" } } diff --git a/os_client_config/vendors/entercloudsuite.json b/os_client_config/vendors/entercloudsuite.json index 5a425b4c1..6d2fc129e 100644 --- a/os_client_config/vendors/entercloudsuite.json +++ b/os_client_config/vendors/entercloudsuite.json @@ -2,8 +2,9 @@ "name": "entercloudsuite", "profile": { "auth": { - "auth_url": "https://api.entercloudsuite.com/v2.0" + "auth_url": "https://api.entercloudsuite.com/" }, + "identity_api_version": "3", "volume_api_version": "1", "regions": [ "it-mil1", diff --git a/os_client_config/vendors/hp.json b/os_client_config/vendors/hp.json index ac280f2d1..b06b90ad5 100644 --- a/os_client_config/vendors/hp.json +++ b/os_client_config/vendors/hp.json @@ -2,12 +2,13 @@ "name": "hp", "profile": { "auth": { - "auth_url": "https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0" + "auth_url": "https://region-b.geo-1.identity.hpcloudsvc.com:35357" }, "regions": [ "region-a.geo-1", "region-b.geo-1" ], + "identity_api_version": "3", "dns_service_type": "hpext:dns", "volume_api_version": "1", "image_api_version": "1" diff --git a/os_client_config/vendors/internap.json b/os_client_config/vendors/internap.json index 9b27536d5..d5ad49f6d 100644 --- a/os_client_config/vendors/internap.json +++ b/os_client_config/vendors/internap.json @@ -2,13 +2,14 @@ "name": "internap", "profile": { "auth": { - "auth_url": "https://identity.api.cloud.iweb.com/v2.0" + "auth_url": "https://identity.api.cloud.iweb.com" }, "regions": [ "ams01", "da01", "nyj01" ], + "identity_api_version": "3", "image_api_version": "1", "floating_ip_source": "None" } diff --git a/os_client_config/vendors/ovh.json b/os_client_config/vendors/ovh.json index 032741f83..664f1617f 100644 --- a/os_client_config/vendors/ovh.json +++ b/os_client_config/vendors/ovh.json @@ -2,13 +2,14 @@ "name": "ovh", "profile": { "auth": { - "auth_url": "https://auth.cloud.ovh.net/v2.0" + "auth_url": "https://auth.cloud.ovh.net/" }, "regions": [ "BHS1", "GRA1", "SBG1" ], + "identity_api_version": "3", "image_format": "raw", "floating_ip_source": "None" } diff --git a/os_client_config/vendors/runabove.json b/os_client_config/vendors/runabove.json index 56dd9453c..abf111633 100644 --- a/os_client_config/vendors/runabove.json +++ b/os_client_config/vendors/runabove.json @@ -2,12 +2,13 @@ "name": "runabove", "profile": { "auth": { - "auth_url": "https://auth.runabove.io/v2.0" + "auth_url": "https://auth.runabove.io/" }, "regions": [ "BHS-1", "SBG-1" ], + "identity_api_version": "3", "image_format": "qcow2", "floating_ip_source": "None" } diff --git a/os_client_config/vendors/ultimum.json b/os_client_config/vendors/ultimum.json index 0b38d71db..4bfd088cd 100644 --- a/os_client_config/vendors/ultimum.json +++ b/os_client_config/vendors/ultimum.json @@ -2,8 +2,9 @@ "name": "ultimum", "profile": { "auth": { - "auth_url": "https://console.ultimum-cloud.com:5000/v2.0" + "auth_url": "https://console.ultimum-cloud.com:5000/" }, + "identity_api_version": "3", "volume_api_version": "1", "region-name": "RegionOne" } From 0b270f0bc9f6dd31d9c17bcc4d49d15630ee999b Mon Sep 17 00:00:00 2001 From: LiuNanke Date: Wed, 6 Jan 2016 22:49:52 +0800 Subject: [PATCH 232/365] Replace assertEqual(None, *) with assertIsNone in tests Replace assertEqual(None, *) with assertIsNone in tests to have more clear messages in case of failure. There have one more place should be modified. Change-Id: I53a8f129db0108892b8377edce2dbf19b0b95f5d Closes-bug: #1280522 --- os_client_config/tests/test_cloud_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 01581b1fb..9d6011120 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -148,8 +148,7 @@ class TestCloudConfig(base.TestCase): self.assertEqual('volume', cc.get_service_type('volume')) self.assertEqual('http://compute.example.com', cc.get_endpoint('compute')) - self.assertEqual(None, - cc.get_endpoint('image')) + self.assertIsNone(cc.get_endpoint('image')) self.assertIsNone(cc.get_service_name('compute')) self.assertEqual('locks', cc.get_service_name('identity')) From cab0469ec4471a5fe924d6049cbfcdf2ac0cdba4 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 5 Jan 2016 09:08:05 -0600 Subject: [PATCH 233/365] Add IBM Public Cloud IBM Cloud has a public Openstack Cloud. We should support it. Change-Id: If0bc29c41869494b2a4da944f7792cbe0f217f0e --- doc/source/vendor-support.rst | 13 +++++++++++++ os_client_config/vendors/ibmcloud.json | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 os_client_config/vendors/ibmcloud.json diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index a215822da..46c95d8c4 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -159,6 +159,19 @@ region-b.geo-1 US East * Public IPv4 is provided via NAT with Neutron Floating IP * Volume API Version is 1 +ibmcloud +-------- + +https://identity.open.softlayer.com + +============== ================ +Region Name Human Name +============== ================ +london London, UK +============== ================ + +* Public IPv4 is provided via NAT with Neutron Floating IP + internap -------- diff --git a/os_client_config/vendors/ibmcloud.json b/os_client_config/vendors/ibmcloud.json new file mode 100644 index 000000000..90962c60e --- /dev/null +++ b/os_client_config/vendors/ibmcloud.json @@ -0,0 +1,13 @@ +{ + "name": "ibmcloud", + "profile": { + "auth": { + "auth_url": "https://identity.open.softlayer.com" + }, + "volume_api_version": "2", + "identity_api_version": "3", + "regions": [ + "london" + ] + } +} From caae8ad43487d5060d113d294c8d8862c7d3f788 Mon Sep 17 00:00:00 2001 From: LiuNanke Date: Thu, 7 Jan 2016 15:21:31 +0800 Subject: [PATCH 234/365] Remove openstack-common.conf We don't sync from oslo-incubator, so don't need this file any more. Change-Id: Ia4acc67fe38c4a27a098c4da263265ed3742b7e7 --- openstack-common.conf | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 openstack-common.conf diff --git a/openstack-common.conf b/openstack-common.conf deleted file mode 100644 index e8eb2aa2c..000000000 --- a/openstack-common.conf +++ /dev/null @@ -1,6 +0,0 @@ -[DEFAULT] - -# The list of modules to copy from oslo-incubator.git - -# The base module to hold the copy of openstack.common -base=os_client_config \ No newline at end of file From 9835daf9f684556c5aed4834dc086e932788f9bc Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 8 Jan 2016 20:24:17 -0500 Subject: [PATCH 235/365] Add barbicanclient support barbicanclient is a lovely client library, so we should add support for make_legacy_client to doing the right things constructing a Client object. Change-Id: Idf015b1119ef76b951c195a6498cbb7a928d6e44 --- os_client_config/cloud_config.py | 7 +++++-- os_client_config/constructors.json | 1 + os_client_config/defaults.json | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index f73da04ed..3b9bee98f 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -305,7 +305,7 @@ class CloudConfig(object): endpoint_override = self.get_endpoint(service_key) if not interface_key: - if service_key == 'image': + if service_key in ('image', 'key-manager'): interface_key = 'interface' else: interface_key = 'endpoint_type' @@ -348,7 +348,10 @@ class CloudConfig(object): if 'endpoint' not in constructor_kwargs: endpoint = self.get_session_endpoint('identity') constructor_kwargs['endpoint'] = endpoint - constructor_args.append(version) + if service_key == 'key-manager': + constructor_kwargs['version'] = version + else: + constructor_args.append(version) return client_class(*constructor_args, **constructor_kwargs) diff --git a/os_client_config/constructors.json b/os_client_config/constructors.json index be4433920..89c844c55 100644 --- a/os_client_config/constructors.json +++ b/os_client_config/constructors.json @@ -3,6 +3,7 @@ "database": "troveclient.client.Client", "identity": "keystoneclient.client.Client", "image": "glanceclient.Client", + "key-manager": "barbicanclient.client.Client", "metering": "ceilometerclient.client.Client", "network": "neutronclient.neutron.client.Client", "object-store": "swiftclient.client.Connection", diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index 2ffb67262..f501862d5 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -12,6 +12,7 @@ "image_api_use_tasks": false, "image_api_version": "2", "image_format": "qcow2", + "key_manager_api_version": "v1", "metering_api_version": "2", "network_api_version": "2", "object_store_api_version": "1", From f61a487fa13c8292b9fd3ac103e1133ac05dbd26 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 8 Jan 2016 20:38:35 -0500 Subject: [PATCH 236/365] Use _get_client in make_client helper function We have a capability to know what constructor is needed for make_client, but we didn't plumb it in. Make sure that the only thing needed is: os_client_config.make_client('compute') Change-Id: I02aa1c46fa7cdfdb1409f8e1232e364b5ba48cd2 --- os_client_config/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index ece155983..52fcb8511 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -14,6 +14,7 @@ import sys +from os_client_config import cloud_config from os_client_config.config import OpenStackConfig # noqa @@ -34,7 +35,7 @@ def simple_client(service_key, cloud=None, region_name=None): cloud=cloud, region_name=region_name).get_session_client(service_key) -def make_client(service_key, constructor, options=None, **kwargs): +def make_client(service_key, constructor=None, options=None, **kwargs): """Simple wrapper for getting a client instance from a client lib. OpenStack Client Libraries all have a fairly consistent constructor @@ -44,6 +45,8 @@ def make_client(service_key, constructor, options=None, **kwargs): variables and clouds.yaml - and takes as **kwargs anything you'd expect to pass in. """ + if not constructor: + constructor = cloud_config._get_client(service_key) config = OpenStackConfig() if options: config.register_argparse_options(options, sys.argv, service_key) @@ -51,5 +54,5 @@ def make_client(service_key, constructor, options=None, **kwargs): else: parsed_options = None - cloud_config = config.get_one_cloud(options=parsed_options, **kwargs) - return cloud_config.get_legacy_client(service_key, constructor) + cloud = config.get_one_cloud(options=parsed_options, **kwargs) + return cloud.get_legacy_client(service_key, constructor) From cd5f16cc4d78fde5a812e2715ee9db430760972f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 8 Jan 2016 20:50:35 -0500 Subject: [PATCH 237/365] Pass version arg by name not position Everyone except neutron has a first parameter called "version" - so we can pass it by name. For neutron, add a workaround, becuase YAY people being different. Change-Id: Icfd92e5e31763ffccc1ff673298f89d1888941fe --- os_client_config/cloud_config.py | 9 ++++----- os_client_config/tests/test_cloud_config.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 3b9bee98f..85c6f2a0b 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -335,7 +335,6 @@ class CloudConfig(object): constructor_kwargs['endpoint_override'] = endpoint constructor_kwargs.update(kwargs) constructor_kwargs[interface_key] = interface - constructor_args = [] if pass_version_arg: if not version: version = self.get_api_version(service_key) @@ -348,12 +347,12 @@ class CloudConfig(object): if 'endpoint' not in constructor_kwargs: endpoint = self.get_session_endpoint('identity') constructor_kwargs['endpoint'] = endpoint - if service_key == 'key-manager': - constructor_kwargs['version'] = version + if service_key == 'network': + constructor_kwargs['api_version'] = version else: - constructor_args.append(version) + constructor_kwargs['version'] = version - return client_class(*constructor_args, **constructor_kwargs) + return client_class(**constructor_kwargs) def _get_swift_client(self, client_class, **kwargs): session = self.get_session() diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 9d6011120..a01d0e1b5 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -304,7 +304,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('image', mock_client) mock_client.assert_called_with( - 2.0, + version=2.0, service_name=None, endpoint_override='http://example.com', region_name='region-al', @@ -325,7 +325,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('image', mock_client) mock_client.assert_called_with( - 2.0, + version=2.0, service_name=None, endpoint_override='http://example.com/override', region_name='region-al', @@ -347,7 +347,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('image', mock_client) mock_client.assert_called_with( - 2.0, + version=2.0, service_name=None, endpoint_override='http://example.com', region_name='region-al', @@ -369,7 +369,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('image', mock_client) mock_client.assert_called_with( - '1', + version='1', service_name=None, endpoint_override='http://example.com', region_name='region-al', @@ -391,7 +391,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('image', mock_client, version='beef') mock_client.assert_called_with( - 'beef', + version='beef', service_name=None, endpoint_override='http://example.com', region_name='region-al', @@ -411,7 +411,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('network', mock_client) mock_client.assert_called_with( - '2.0', + api_version='2.0', endpoint_type='public', endpoint_override=None, region_name='region-al', @@ -429,7 +429,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('compute', mock_client) mock_client.assert_called_with( - '2', + version='2', endpoint_type='public', endpoint_override='http://compute.example.com', region_name='region-al', @@ -447,7 +447,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('identity', mock_client) mock_client.assert_called_with( - '2.0', + version='2.0', endpoint='http://example.com/v2', endpoint_type='admin', endpoint_override=None, @@ -467,7 +467,7 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('identity', mock_client) mock_client.assert_called_with( - '3', + version='3', endpoint='http://example.com', endpoint_type='admin', endpoint_override=None, From 7e5496763522475bb07a377359d69454f1942e1b Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 4 Jan 2016 12:56:28 -0600 Subject: [PATCH 238/365] Return empty dict instead of None for lack of file We return None for the file content for non-existent files as a fallback. This is normally fine, but in the case of a person having _only_ a secure.conf file, this means that the dictionary merge fails. Change-Id: I61cc0a8c709ea3510428fc3dfce63dc254c07c83 --- os_client_config/config.py | 2 +- os_client_config/tests/test_config.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index d490006b4..d3663077c 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -315,7 +315,7 @@ class OpenStackConfig(object): return path, json.load(f) else: return path, yaml.safe_load(f) - return (None, None) + return (None, {}) def _normalize_keys(self, config): new_config = {} diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 4440ac8e1..dce436af6 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -171,6 +171,13 @@ class TestConfig(base.TestCase): self.assertEqual('user', cc.auth['username']) self.assertEqual('testpass', cc.auth['password']) + def test_only_secure_yaml(self): + c = config.OpenStackConfig(config_files=['nonexistent'], + vendor_files=['nonexistent'], + secure_files=[self.secure_yaml]) + cc = c.get_one_cloud(cloud='_test_cloud_no_vendor') + self.assertEqual('testpass', cc.auth['password']) + def test_get_cloud_names(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], secure_files=[self.no_yaml]) From a8532f6c8d221628b697ddb0d134e2a000ef61d6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 13 Jan 2016 13:37:14 -0500 Subject: [PATCH 239/365] Fix a precedence problem with auth arguments With the current code, OS_TENANT_NAME will take precednece over --os-project-name beause OS_TENANT_NAME gets early-moved to config['auth']['project_name'], then when the argparse value gets put into config['project_name'] the auth fixing sees auth['project_name'] and thinks it should win. Change-Id: I97084ea221eb963f14d98cf550a04bbd5c7d954c --- os_client_config/config.py | 4 +++- os_client_config/tests/test_config.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index d3663077c..7e56f61b8 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -447,7 +447,7 @@ class OpenStackConfig(object): if 'cloud' in cloud: del cloud['cloud'] - return self._fix_backwards_madness(cloud) + return cloud def _expand_vendor_profile(self, name, cloud, our_cloud): # Expand a profile if it exists. 'cloud' is an old confusing name @@ -896,6 +896,8 @@ class OpenStackConfig(object): if 'endpoint_type' in config: config['interface'] = config.pop('endpoint_type') + config = self._fix_backwards_madness(config) + for key in BOOL_KEYS: if key in config: if type(config[key]) is not bool: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index dce436af6..1a16bd8ce 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -473,6 +473,16 @@ class TestConfigArgparse(base.TestCase): opts, _remain = parser.parse_known_args(['--os-cloud', 'foo']) self.assertEqual(opts.os_cloud, 'foo') + def test_env_argparse_precedence(self): + self.useFixture(fixtures.EnvironmentVariable( + 'OS_TENANT_NAME', 'tenants-are-bad')) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + cc = c.get_one_cloud( + cloud='envvars', argparse=self.options) + self.assertEqual(cc.auth['project_name'], 'project') + def test_argparse_default_no_token(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From 2f1d184a8e6bed99d027b5ec9f8ee475e527cbdc Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 14 Jan 2016 17:10:21 +0000 Subject: [PATCH 240/365] set up release notes build Add release notes build files and tox environment so the existing release notes job has something to build from. Change-Id: I717d4e7af438cbc94eecf32472f6d1f8213761b8 --- releasenotes/source/_static/.placeholder | 0 releasenotes/source/_templates/.placeholder | 0 releasenotes/source/conf.py | 261 ++++++++++++++++++++ releasenotes/source/index.rst | 17 ++ releasenotes/source/unreleased.rst | 5 + tox.ini | 5 +- 6 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 releasenotes/source/_static/.placeholder create mode 100644 releasenotes/source/_templates/.placeholder create mode 100644 releasenotes/source/conf.py create mode 100644 releasenotes/source/index.rst create mode 100644 releasenotes/source/unreleased.rst diff --git a/releasenotes/source/_static/.placeholder b/releasenotes/source/_static/.placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/releasenotes/source/_templates/.placeholder b/releasenotes/source/_templates/.placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 000000000..282dbd784 --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,261 @@ +# -*- coding: utf-8 -*- +# +# Os-Client-Config Release Notes documentation build configuration file, created by +# sphinx-quickstart on Thu Nov 5 11:50:32 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'oslosphinx', + 'reno.sphinxext', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'os-client-config Release Notes' +copyright = u'2015, os-client-config developers' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +import pbr.version +occ_version = pbr.version.VersionInfo('os-client-config') +# The short X.Y version. +version = occ_version.canonical_version_string() +# The full version, including alpha/beta/rc tags. +release = occ_version.version_string_with_vcs() + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'OCCReleaseNotesdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'OCCReleaseNotes.tex', u'os-client-config Release Notes Documentation', + u'os-client-config developers', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'occreleasenotes', u'os-client-config Release Notes Documentation', + [u'os-client-config developers'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'OCCReleaseNotes', u'os-client-config Release Notes Documentation', + u'os-client-config developers', 'OCCReleaseNotes', + 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 000000000..386434ef8 --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,17 @@ +Welcome to Nova Release Notes documentation! +============================================== + +Contents +======== + +.. toctree:: + :maxdepth: 2 + + unreleased + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/releasenotes/source/unreleased.rst b/releasenotes/source/unreleased.rst new file mode 100644 index 000000000..875030f9d --- /dev/null +++ b/releasenotes/source/unreleased.rst @@ -0,0 +1,5 @@ +============================ +Current Series Release Notes +============================ + +.. release-notes:: diff --git a/tox.ini b/tox.ini index 95dff6ba4..617831dfe 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,9 @@ commands = python setup.py build_sphinx python setup.py check -r -s +[testenv:releasenotes] +commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html + [flake8] # H803 skipped on purpose per list discussion. # E123, E125 skipped as they are invalid PEP-8. @@ -35,4 +38,4 @@ commands = show-source = True ignore = E123,E125,H803 builtins = _ -exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,releasenotes/source/conf.py From cfd29196fedf41dcd61d0df6b0109dc8e43abfc8 Mon Sep 17 00:00:00 2001 From: LiuNanke Date: Thu, 14 Jan 2016 16:45:59 +0800 Subject: [PATCH 241/365] Clean up removed hacking rule from [flake8] ignore lists We bump hacking>=0.10.2, and hacking removed some rules, for the full list of rules please see [1]. So don't need them any more. Hacking related commits: Remove H904 in commit b1fe19ebebe47a36b905d709467f5e82521bbd96 Remove H803 in commit f01ce4fd822546cbd52a0aedc49184bddbfe1b10 Remove H307 in commit ec4833b206c23b0b6f9c6b101c70ab925a5e9c67 Remove H305 in commit 8f1fcbdb9aa4fc61349e5e879153c722195b1233 [1]https://github.com/openstack-dev/hacking/blob/master/setup.cfg#L30 Change-Id: I24b82c1913d3d42cc5228b1db700b787623fcdc5 --- test-requirements.txt | 2 +- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index a50a202e3..5e4c30446 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=0.9.2,<0.10 +hacking>=0.10.2,<0.11 # Apache-2.0 coverage>=3.6 extras diff --git a/tox.ini b/tox.ini index 617831dfe..c3f42d6ac 100644 --- a/tox.ini +++ b/tox.ini @@ -32,10 +32,10 @@ commands = commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [flake8] -# H803 skipped on purpose per list discussion. # E123, E125 skipped as they are invalid PEP-8. show-source = True -ignore = E123,E125,H803 +ignore = E123,E125 builtins = _ exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,releasenotes/source/conf.py + From a2db877b41fad494fe9daa09b5c77914638ac605 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 19 Jan 2016 10:12:02 -0500 Subject: [PATCH 242/365] Don't set project_domain if not project scoped The code to expand domain_{name,id} to {user,project}_domain_{name,id} is flawed in that it sets a project_domain_{name,id} even if a project_{name,id} is not set. There is a valid use case for not having a project_{name,id} - specifically getting a domain-scoped token. In the case where we do not set a project, check for that and don't make further assumptions that the domain input needs to be "fixed". Closes-Bug: #1535676 Change-Id: I825fe4bc375687208bb176bb5990c23fe87c8f9d --- os_client_config/config.py | 9 +++++++++ os_client_config/tests/base.py | 9 +++++++++ os_client_config/tests/test_config.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 378cd3b60..0444316ed 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -480,6 +480,11 @@ class OpenStackConfig(object): cloud = self._handle_domain_id(cloud) return cloud + def _project_scoped(self, cloud): + return ('project_id' in cloud or 'project_name' in cloud + or 'project_id' in cloud['auth'] + or 'project_name' in cloud['auth']) + def _handle_domain_id(self, cloud): # Allow people to just specify domain once if it's the same mappings = { @@ -487,6 +492,10 @@ class OpenStackConfig(object): 'domain_name': ('user_domain_name', 'project_domain_name'), } for target_key, possible_values in mappings.items(): + if not self._project_scoped(cloud): + if target_key in cloud and target_key not in cloud['auth']: + cloud['auth'][target_key] = cloud.pop(target_key) + continue for key in possible_values: if target_key in cloud['auth'] and key not in cloud['auth']: cloud['auth'][key] = cloud['auth'][target_key] diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 3f00f6d82..9b784b157 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -73,6 +73,7 @@ USER_CONF = { 'auth': { 'username': 'testuser', 'password': 'testpass', + 'domain_id': 'awesome-domain', 'project_id': 12345, 'auth_url': 'http://example.com/v2', }, @@ -128,6 +129,14 @@ USER_CONF = { 'password': 'testpass', }, }, + '_test-cloud-domain-scoped_': { + 'auth': { + 'auth_url': 'http://example.com/v2', + 'username': 'testuser', + 'password': 'testpass', + 'domain-id': '12345', + }, + }, }, 'ansible': { 'expand-hostvars': False, diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 1a16bd8ce..73be114a5 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -103,6 +103,22 @@ class TestConfig(base.TestCase): self.assertNotIn('domain_id', cc.auth) self.assertNotIn('domain-id', cc.auth) + def test_get_one_cloud_domain_scoped(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test-cloud-domain-scoped_') + self.assertEqual('12345', cc.auth['domain_id']) + self.assertNotIn('user_domain_id', cc.auth) + self.assertNotIn('project_domain_id', cc.auth) + + def test_get_one_cloud_infer_user_domain(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test-cloud-int-project_') + self.assertEqual('awesome-domain', cc.auth['user_domain_id']) + self.assertEqual('awesome-domain', cc.auth['project_domain_id']) + self.assertNotIn('domain_id', cc.auth) + def test_get_one_cloud_with_hyphenated_project_id(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) @@ -183,6 +199,7 @@ class TestConfig(base.TestCase): secure_files=[self.no_yaml]) self.assertEqual( ['_test-cloud-domain-id_', + '_test-cloud-domain-scoped_', '_test-cloud-int-project_', '_test-cloud_', '_test-cloud_no_region', From ae8f4b65e3ac460b7764f2cdf9dfdcfe41ee0d22 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 17 Jan 2016 09:08:27 -0500 Subject: [PATCH 243/365] Go ahead and remove final excludes os-client-config is clean on E125 and E123 is ignored in recent pep8 by default. Also, even though they are not 'valid' pep8 rules, they are actually both nice styles and consistent with how we code os-client-config anyway. Change-Id: I7764e1511ed580d37b9a0a8be6743a5fa50441e5 --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index c3f42d6ac..3df64e24c 100644 --- a/tox.ini +++ b/tox.ini @@ -32,10 +32,7 @@ commands = commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [flake8] -# E123, E125 skipped as they are invalid PEP-8. - show-source = True -ignore = E123,E125 builtins = _ exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,releasenotes/source/conf.py From 42727a5e182eade19f4007195ee9058e56ba27bc Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 23 Jan 2016 13:03:55 -0500 Subject: [PATCH 244/365] Stop ignoring v2password plugin We have no codepaths that currently set v2password plugin by default. However, there are some cases, such as old clouds, where a user needs to explicitly set v2password as the auth_type to avoid version discovery because their cloud is old enough to not support it. If the user sets v2password, keep it and align the auth parameters the other direction to set tenant_name and tenant_id. Co-Authored-By: David Shrewsbury Change-Id: Ib9eb3ae163b79b67737d01868868187b6dee1756 --- os_client_config/config.py | 21 +++++++++----- os_client_config/tests/test_config.py | 40 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 0444316ed..1c47ed72a 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -474,8 +474,8 @@ class OpenStackConfig(object): name)) def _fix_backwards_madness(self, cloud): - cloud = self._fix_backwards_project(cloud) cloud = self._fix_backwards_auth_plugin(cloud) + cloud = self._fix_backwards_project(cloud) cloud = self._fix_backwards_interface(cloud) cloud = self._handle_domain_id(cloud) return cloud @@ -507,10 +507,6 @@ class OpenStackConfig(object): # Also handle moving domain names into auth so that domain mapping # is easier mappings = { - 'project_id': ('tenant_id', 'tenant-id', - 'project_id', 'project-id'), - 'project_name': ('tenant_name', 'tenant-name', - 'project_name', 'project-name'), 'domain_id': ('domain_id', 'domain-id'), 'domain_name': ('domain_name', 'domain-name'), 'user_domain_id': ('user_domain_id', 'user-domain-id'), @@ -520,6 +516,19 @@ class OpenStackConfig(object): 'project_domain_name', 'project-domain-name'), 'token': ('auth-token', 'auth_token', 'token'), } + if cloud.get('auth_type', None) == 'v2password': + # If v2password is explcitly requested, this is to deal with old + # clouds. That's fine - we need to map settings in the opposite + # direction + mappings['tenant_id'] = ( + 'project_id', 'project-id', 'tenant_id', 'tenant-id') + mappings['tenant_name'] = ( + 'project_name', 'project-name', 'tenant_name', 'tenant-name') + else: + mappings['project_id'] = ( + 'tenant_id', 'tenant-id', 'project_id', 'project-id') + mappings['project_name'] = ( + 'tenant_name', 'tenant-name', 'project_name', 'project-name') for target_key, possible_values in mappings.items(): target = None for key in possible_values: @@ -549,8 +558,6 @@ class OpenStackConfig(object): # use of the auth plugin that can do auto-selection and dealing # with that based on auth parameters. v2password is basically # completely broken - if cloud['auth_type'] == 'v2password': - cloud['auth_type'] = 'password' return cloud def register_argparse_arguments(self, parser, argv, service_keys=[]): diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 73be114a5..10a3d7b31 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -713,3 +713,43 @@ class TestBackwardsCompatibility(base.TestCase): 'auth_type': 'v3password', } self.assertDictEqual(expected, result) + + def test_project_v2password(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cloud = { + 'auth_type': 'v2password', + 'auth': { + 'project-name': 'my_project_name', + 'project-id': 'my_project_id' + } + } + result = c._fix_backwards_project(cloud) + expected = { + 'auth_type': 'v2password', + 'auth': { + 'tenant_name': 'my_project_name', + 'tenant_id': 'my_project_id' + } + } + self.assertEqual(expected, result) + + def test_project_password(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cloud = { + 'auth_type': 'password', + 'auth': { + 'project-name': 'my_project_name', + 'project-id': 'my_project_id' + } + } + result = c._fix_backwards_project(cloud) + expected = { + 'auth_type': 'password', + 'auth': { + 'project_name': 'my_project_name', + 'project_id': 'my_project_id' + } + } + self.assertEqual(expected, result) From fe2558a2d5b6a2fa8c2f3f3c5472b79a7e01ba4a Mon Sep 17 00:00:00 2001 From: Steve Martinelli Date: Mon, 25 Jan 2016 20:37:22 -0500 Subject: [PATCH 245/365] Add support for zetta.io zetta has an openstack cloud, let's add support for it. Change-Id: I86cda3e42fff468786b2809bb367ad59241bb397 Closes-Bug: 1537959 --- doc/source/vendor-support.rst | 14 ++++++++++++++ os_client_config/vendors/zetta.json | 13 +++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 os_client_config/vendors/zetta.json diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 46c95d8c4..e007b70f8 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -305,3 +305,17 @@ ca-ymq-1 Montreal * DNS API Version is 1 * Identity API Version is 3 + +zetta +----- + +https://identity.api.zetta.io/v3 + +============== ================ +Region Name Human Name +============== ================ +no-osl1 Oslo +============== ================ + +* DNS API Version is 2 +* Identity API Version is 3 diff --git a/os_client_config/vendors/zetta.json b/os_client_config/vendors/zetta.json new file mode 100644 index 000000000..44e9711ff --- /dev/null +++ b/os_client_config/vendors/zetta.json @@ -0,0 +1,13 @@ +{ + "name": "zetta", + "profile": { + "auth": { + "auth_url": "https://identity.api.zetta.io/v3" + }, + "regions": [ + "no-osl1" + ], + "identity_api_version": "3", + "dns_api_version": "2" + } +} From 8264e09c69bd6c017c1716a70cec21c28919e6d1 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Wed, 10 Feb 2016 11:51:37 -0500 Subject: [PATCH 246/365] Added SSL support for VEXXHOST VEXXHOST cloud uses SSL for Keystone and all other services, change the auth URL to the SSL endpoint. Change-Id: If80c76603de44d005d6af1726f34d924384bf747 --- os_client_config/vendors/vexxhost.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/vendors/vexxhost.json b/os_client_config/vendors/vexxhost.json index dd683be86..aa2cedc68 100644 --- a/os_client_config/vendors/vexxhost.json +++ b/os_client_config/vendors/vexxhost.json @@ -2,7 +2,7 @@ "name": "vexxhost", "profile": { "auth": { - "auth_url": "http://auth.vexxhost.net" + "auth_url": "https://auth.vexxhost.net" }, "regions": [ "ca-ymq-1" From 10a9369062504cce1c4c79a1b112e619082cf6ab Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 10 Feb 2016 11:24:10 -0600 Subject: [PATCH 247/365] Remove HP and RunAbove from vendor profiles HP has already shut down its public cloud. RunAbove is shutting down 17th February as part of the migration to OVH.com. Neither are therefore valid vendors any longer. Change-Id: I8d305ca2b1cbaf67e6711eedaa1a4c5668a42be7 --- doc/source/vendor-support.rst | 31 -------------------------- os_client_config/vendors/hp.json | 16 ------------- os_client_config/vendors/runabove.json | 15 ------------- 3 files changed, 62 deletions(-) delete mode 100644 os_client_config/vendors/hp.json delete mode 100644 os_client_config/vendors/runabove.json diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index e007b70f8..d30c104e7 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -142,23 +142,6 @@ de-fra1 Frankfurt, DE * Volume API Version is 1 -hp --- - -https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 - -============== ================ -Region Name Human Name -============== ================ -region-a.geo-1 US West -region-b.geo-1 US East -============== ================ - -* DNS Service Type is `hpext:dns` -* Image API Version is 1 -* Public IPv4 is provided via NAT with Neutron Floating IP -* Volume API Version is 1 - ibmcloud -------- @@ -233,20 +216,6 @@ SYD Sydney :xenapi_use_agent: False * Volume API Version is 1 -runabove --------- - -https://auth.runabove.io/v2.0 - -============== ================ -Region Name Human Name -============== ================ -SBG-1 Strassbourg, FR -BHS-1 Beauharnois, QC -============== ================ - -* Floating IPs are not supported - switchengines ------------- diff --git a/os_client_config/vendors/hp.json b/os_client_config/vendors/hp.json deleted file mode 100644 index b06b90ad5..000000000 --- a/os_client_config/vendors/hp.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "hp", - "profile": { - "auth": { - "auth_url": "https://region-b.geo-1.identity.hpcloudsvc.com:35357" - }, - "regions": [ - "region-a.geo-1", - "region-b.geo-1" - ], - "identity_api_version": "3", - "dns_service_type": "hpext:dns", - "volume_api_version": "1", - "image_api_version": "1" - } -} diff --git a/os_client_config/vendors/runabove.json b/os_client_config/vendors/runabove.json deleted file mode 100644 index abf111633..000000000 --- a/os_client_config/vendors/runabove.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "runabove", - "profile": { - "auth": { - "auth_url": "https://auth.runabove.io/" - }, - "regions": [ - "BHS-1", - "SBG-1" - ], - "identity_api_version": "3", - "image_format": "qcow2", - "floating_ip_source": "None" - } -} From dd1f03c597daf1dc422608ab8e2b0b6b78168a3f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 10 Feb 2016 17:37:54 -0600 Subject: [PATCH 248/365] Send swiftclient username/password and token For longer-lived operations, tokens can timeout and we need to get new ones. While in theory we should be keystoneauth aware and passing around sessions, swiftclient does not yet support this. So, instead of passing in just a preauthtoken, also pass in credentials if we have them. However, for plugin types that swift does not know about directly, only preauthtoken will be used as before. Change-Id: If724fdcd0649d9fa3b3ee7b127e49a3f77e3b767 --- os_client_config/cloud_config.py | 37 ++++- os_client_config/tests/test_cloud_config.py | 157 ++++++++++++++++++-- 2 files changed, 176 insertions(+), 18 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 85c6f2a0b..b19607e03 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -355,22 +355,53 @@ class CloudConfig(object): return client_class(**constructor_kwargs) def _get_swift_client(self, client_class, **kwargs): + auth_args = self.get_auth_args() + auth_version = self.get_api_version('identity') session = self.get_session() token = session.get_token() endpoint = self.get_session_endpoint(service_key='object-store') if not endpoint: return None + # If we have a username/password, we want to pass them to + # swift - because otherwise it will not re-up tokens appropriately + # However, if we only have non-password auth, then get a token + # and pass it in swift_kwargs = dict( + auth_version=auth_version, preauthurl=endpoint, preauthtoken=token, - auth_version=self.get_api_version('identity'), os_options=dict( + region_name=self.get_region_name(), auth_token=token, object_storage_url=endpoint, - region_name=self.get_region_name()), - ) + service_type=self.get_service_type('object-store'), + endpoint_type=self.get_interface('object-store'), + + )) if self.config['api_timeout'] is not None: swift_kwargs['timeout'] = float(self.config['api_timeout']) + + # create with password + swift_kwargs['user'] = auth_args.get('username') + swift_kwargs['key'] = auth_args.get('password') + swift_kwargs['authurl'] = auth_args.get('auth_url') + os_options = {} + if auth_version == '2.0': + os_options['tenant_name'] = auth_args.get('project_name') + os_options['tenant_id'] = auth_args.get('project_id') + else: + os_options['project_name'] = auth_args.get('project_name') + os_options['project_id'] = auth_args.get('project_id') + + for key in ( + 'user_id', + 'project_domain_id', + 'project_domain_name', + 'user_domain_id', + 'user_domain_name'): + os_options[key] = auth_args.get(key) + swift_kwargs['os_options'].update(os_options) + return client_class(**swift_kwargs) def get_cache_expiration_time(self): diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index a01d0e1b5..7a8b77aed 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -235,10 +235,23 @@ class TestCloudConfig(base.TestCase): region_name='region-al', service_type='orchestration') + @mock.patch.object(cloud_config.CloudConfig, 'get_api_version') + @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') - def test_legacy_client_object_store(self, mock_get_session_endpoint): + def test_legacy_client_object_store_password( + self, + mock_get_session_endpoint, + mock_get_auth_args, + mock_get_api_version): mock_client = mock.Mock() - mock_get_session_endpoint.return_value = 'http://example.com/v2' + mock_get_session_endpoint.return_value = 'http://swift.example.com' + mock_get_api_version.return_value = '3' + mock_get_auth_args.return_value = dict( + username='testuser', + password='testpassword', + project_name='testproject', + auth_url='http://example.com', + ) config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) cc = cloud_config.CloudConfig( @@ -246,19 +259,106 @@ class TestCloudConfig(base.TestCase): cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( preauthtoken=mock.ANY, + auth_version=u'3', + authurl='http://example.com', + key='testpassword', os_options={ 'auth_token': mock.ANY, 'region_name': 'region-al', - 'object_storage_url': 'http://example.com/v2' + 'object_storage_url': 'http://swift.example.com', + 'user_id': None, + 'user_domain_name': None, + 'project_name': 'testproject', + 'project_domain_name': None, + 'project_domain_id': None, + 'project_id': None, + 'service_type': 'object-store', + 'endpoint_type': 'public', + 'user_domain_id': None }, - preauthurl='http://example.com/v2', - auth_version='2.0') + preauthurl='http://swift.example.com', + user='testuser') + @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') - def test_legacy_client_object_store_timeout( - self, mock_get_session_endpoint): + def test_legacy_client_object_store_password_v2( + self, mock_get_session_endpoint, mock_get_auth_args): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://swift.example.com' + mock_get_auth_args.return_value = dict( + username='testuser', + password='testpassword', + project_name='testproject', + auth_url='http://example.com', + ) + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('object-store', mock_client) + mock_client.assert_called_with( + preauthtoken=mock.ANY, + auth_version=u'2.0', + authurl='http://example.com', + key='testpassword', + os_options={ + 'auth_token': mock.ANY, + 'region_name': 'region-al', + 'object_storage_url': 'http://swift.example.com', + 'user_id': None, + 'user_domain_name': None, + 'tenant_name': 'testproject', + 'project_domain_name': None, + 'project_domain_id': None, + 'tenant_id': None, + 'service_type': 'object-store', + 'endpoint_type': 'public', + 'user_domain_id': None + }, + preauthurl='http://swift.example.com', + user='testuser') + + @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_object_store( + self, mock_get_session_endpoint, mock_get_auth_args): mock_client = mock.Mock() mock_get_session_endpoint.return_value = 'http://example.com/v2' + mock_get_auth_args.return_value = {} + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock()) + cc.get_legacy_client('object-store', mock_client) + mock_client.assert_called_with( + preauthtoken=mock.ANY, + auth_version=u'2.0', + authurl=None, + key=None, + os_options={ + 'auth_token': mock.ANY, + 'region_name': 'region-al', + 'object_storage_url': 'http://example.com/v2', + 'user_id': None, + 'user_domain_name': None, + 'tenant_name': None, + 'project_domain_name': None, + 'project_domain_id': None, + 'tenant_id': None, + 'service_type': 'object-store', + 'endpoint_type': 'public', + 'user_domain_id': None + }, + preauthurl='http://example.com/v2', + user=None) + + @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') + @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') + def test_legacy_client_object_store_timeout( + self, mock_get_session_endpoint, mock_get_auth_args): + mock_client = mock.Mock() + mock_get_session_endpoint.return_value = 'http://example.com/v2' + mock_get_auth_args.return_value = {} config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) config_dict['api_timeout'] = 9 @@ -267,32 +367,59 @@ class TestCloudConfig(base.TestCase): cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( preauthtoken=mock.ANY, + auth_version=u'2.0', + authurl=None, + key=None, os_options={ 'auth_token': mock.ANY, 'region_name': 'region-al', - 'object_storage_url': 'http://example.com/v2' + 'object_storage_url': 'http://example.com/v2', + 'user_id': None, + 'user_domain_name': None, + 'tenant_name': None, + 'project_domain_name': None, + 'project_domain_id': None, + 'tenant_id': None, + 'service_type': 'object-store', + 'endpoint_type': 'public', + 'user_domain_id': None }, preauthurl='http://example.com/v2', - auth_version='2.0', - timeout=9.0) + timeout=9.0, + user=None) - def test_legacy_client_object_store_endpoint(self): + @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') + def test_legacy_client_object_store_endpoint( + self, mock_get_auth_args): mock_client = mock.Mock() + mock_get_auth_args.return_value = {} config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) - config_dict['object_store_endpoint'] = 'http://example.com/v2' + config_dict['object_store_endpoint'] = 'http://example.com/swift' cc = cloud_config.CloudConfig( "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( preauthtoken=mock.ANY, + auth_version=u'2.0', + authurl=None, + key=None, os_options={ 'auth_token': mock.ANY, 'region_name': 'region-al', - 'object_storage_url': 'http://example.com/v2' + 'object_storage_url': 'http://example.com/swift', + 'user_id': None, + 'user_domain_name': None, + 'tenant_name': None, + 'project_domain_name': None, + 'project_domain_id': None, + 'tenant_id': None, + 'service_type': 'object-store', + 'endpoint_type': 'public', + 'user_domain_id': None }, - preauthurl='http://example.com/v2', - auth_version='2.0') + preauthurl='http://example.com/swift', + user=None) @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') def test_legacy_client_image(self, mock_get_session_endpoint): From 7865abc22b7289b2679f6848395d4850d544d1f0 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 17 Feb 2016 11:46:57 -0800 Subject: [PATCH 249/365] Add release notes Catch up the release notes from the previous release to current state. This does not catch up from the beginning of oscc's history. Change-Id: Ic981fdfbb79cd7fc70167091bdfed281c11eff03 --- ...tch-up-release-notes-e385fad34e9f3d6e.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 releasenotes/notes/catch-up-release-notes-e385fad34e9f3d6e.yaml diff --git a/releasenotes/notes/catch-up-release-notes-e385fad34e9f3d6e.yaml b/releasenotes/notes/catch-up-release-notes-e385fad34e9f3d6e.yaml new file mode 100644 index 000000000..e7b98afe3 --- /dev/null +++ b/releasenotes/notes/catch-up-release-notes-e385fad34e9f3d6e.yaml @@ -0,0 +1,22 @@ +--- +prelude: > + Swiftclient instantiation now provides authentication + information so that long lived swiftclient objects can + reauthenticate if necessary. This should be a temporary + situation until swiftclient supports keystoneauth + sessions at which point os-client-config will instantiate + swiftclient with a keystoneauth session. +features: + - Swiftclient instantiation now provides authentication + information so that long lived swiftclient objects can + reauthenticate if necessary. + - Add support for explicit v2password auth type. + - Add SSL support to VEXXHOST vendor profile. + - Add zetta.io cloud vendor profile. +fixes: + - Fix bug where project_domain_{name,id} was set even + if project_{name,id} was not set. +other: + - HPCloud vendor profile removed due to cloud shutdown. + - RunAbove vendor profile removed due to migration to + OVH. From 35ece66b4cc263816b9b28239901a5cbad61c8eb Mon Sep 17 00:00:00 2001 From: Arie Bregman Date: Thu, 18 Feb 2016 14:23:12 +0200 Subject: [PATCH 250/365] Fix formulation Fixed the formulation for the message that appears when cloud config couldn't be found. Change-Id: I1a4a83fe598d4eab52713061471fab8d1c46ec91 --- os_client_config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1c47ed72a..2f6adeb2b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -432,7 +432,7 @@ class OpenStackConfig(object): # Only validate cloud name if one was given if name and name not in self.cloud_config['clouds']: raise exceptions.OpenStackConfigException( - "Named cloud {name} requested that was not found.".format( + "Cloud {name} was not found.".format( name=name)) our_cloud = self.cloud_config['clouds'].get(name, dict()) From 7a4993da4190d92cb2f023d84e0e9311db38f0bf Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 22 Feb 2016 06:30:31 -0800 Subject: [PATCH 251/365] Allow session_client to take the same args as make_client make_client is a great, simple yet flexible way to get a fully featured Client object. simple_client is similar for Session objects, but lacks the argparse and arbitrary kwargs that make_client - plus it has a weird name. Since adding those two features to make_client did not make it too confusing - do the same for simple_client. Also, rename it to session_client (with a backwards-compat alias) and add it to the README docs. In the process of doing this, extract the "get me a cloud config" functinality into an additional helper function - get_config. Change-Id: Iadd24dfa021f870b3e5858bab8cd91fc96a373c2 --- README.rst | 17 +++++++++-- os_client_config/__init__.py | 28 +++++++++++-------- .../session-client-b581a6e5d18c8f04.yaml | 6 ++++ 3 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 releasenotes/notes/session-client-b581a6e5d18c8f04.yaml diff --git a/README.rst b/README.rst index f078f3cee..54a5631ed 100644 --- a/README.rst +++ b/README.rst @@ -362,8 +362,6 @@ will get you a fully configured `novaclient` instance. .. code-block:: python - import argparse - import os_client_config nova = os_client_config.make_client('compute') @@ -382,6 +380,21 @@ If you want to do the same thing but also support command line parsing. If you want to get fancier than that in your python, then the rest of the API is available to you. But often times, you just want to do the one thing. +Constructing Mounted Session Objects +------------------------------------ + +What if you want to make direct REST calls via a Session interface? You're +in luck. The same interface for `make_client` is supported for `session_client` +and will return you a keystoneauth Session object that is mounted on the +endpoint for the service you're looking for. + + import os_client_config + + session = os_client_config.session_client('compute', cloud='vexxhost') + + response = session.get('/servers') + server_list = response.json()['servers'] + Source ------ diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index 52fcb8511..6f78d2fca 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -18,7 +18,18 @@ from os_client_config import cloud_config from os_client_config.config import OpenStackConfig # noqa -def simple_client(service_key, cloud=None, region_name=None): +def get_config(service_key=None, options=None, **kwargs): + config = OpenStackConfig() + if options: + config.register_argparse_options(options, sys.argv, service_key) + parsed_options = options.parse_known_args(sys.argv) + else: + parsed_options = None + + return config.get_one_cloud(options=parsed_options, **kwargs) + + +def session_client(service_key, options=None, **kwargs): """Simple wrapper function. It has almost no features. This will get you a raw requests Session Adapter that is mounted @@ -31,8 +42,10 @@ def simple_client(service_key, cloud=None, region_name=None): get_session_client on it. This function is to make it easy to poke at OpenStack REST APIs with a properly configured keystone session. """ - return OpenStackConfig().get_one_cloud( - cloud=cloud, region_name=region_name).get_session_client(service_key) + cloud = get_config(service_key=service_key, options=options, **kwargs) + return cloud.get_session_client(service_key) +# Backwards compat - simple_client was a terrible name +simple_client = session_client def make_client(service_key, constructor=None, options=None, **kwargs): @@ -45,14 +58,7 @@ def make_client(service_key, constructor=None, options=None, **kwargs): variables and clouds.yaml - and takes as **kwargs anything you'd expect to pass in. """ + cloud = get_config(service_key=service_key, options=options, **kwargs) if not constructor: constructor = cloud_config._get_client(service_key) - config = OpenStackConfig() - if options: - config.register_argparse_options(options, sys.argv, service_key) - parsed_options = options.parse_args(sys.argv) - else: - parsed_options = None - - cloud = config.get_one_cloud(options=parsed_options, **kwargs) return cloud.get_legacy_client(service_key, constructor) diff --git a/releasenotes/notes/session-client-b581a6e5d18c8f04.yaml b/releasenotes/notes/session-client-b581a6e5d18c8f04.yaml new file mode 100644 index 000000000..11219016b --- /dev/null +++ b/releasenotes/notes/session-client-b581a6e5d18c8f04.yaml @@ -0,0 +1,6 @@ +--- +features: + - Added kwargs and argparse processing for session_client. +deprecations: + - Renamed simple_client to session_client. simple_client + will remain as an alias for backwards compat. From 03d5659d8b45b0450a86956147cc2d1f27be66ac Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 22 Feb 2016 06:37:13 -0800 Subject: [PATCH 252/365] Update the README a bit Cleaned up example references to now-not-existing HP Public Cloud. Also added a named-cloud entry to the make_client section. Change-Id: I398c438e22eb84d6079a5c45f068753c3bcaa216 --- README.rst | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index 54a5631ed..2e584bd53 100644 --- a/README.rst +++ b/README.rst @@ -88,23 +88,21 @@ An example config file is probably helpful: .. code-block:: yaml clouds: - mordred: - profile: hp + mtvexx: + profile: vexxhost auth: username: mordred@inaugust.com password: XXXXXXXXX project_name: mordred@inaugust.com - region_name: region-b.geo-1 - dns_service_type: hpext:dns - compute_api_version: 1.1 - monty: + region_name: ca-ymq-1 + dns_api_version: 1 + mordred: + region_name: RegionOne auth: - auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0 - username: monty.taylor@hp.com - password: XXXXXXXX - project_name: monty.taylor@hp.com-default-tenant - region_name: region-b.geo-1 - dns_service_type: hpext:dns + username: 'mordred' + password: XXXXXXX + project_name: 'shade' + auth_url: 'https://montytaylor-sjc.openstack.blueboxgrid.com:5001/v2.0' infra: profile: rackspace auth: @@ -221,14 +219,14 @@ are connecting to OpenStack can share a cache should you desire. server: 5 flavor: -1 clouds: - mordred: - profile: hp + mtvexx: + profile: vexxhost auth: username: mordred@inaugust.com password: XXXXXXXXX project_name: mordred@inaugust.com - region_name: region-b.geo-1 - dns_service_type: hpext:dns + region_name: ca-ymq-1 + dns_api_version: 1 IPv6 @@ -247,13 +245,14 @@ environment variable. client: force_ipv4: true clouds: - mordred: - profile: hp + mtvexx: + profile: vexxhost auth: username: mordred@inaugust.com password: XXXXXXXXX project_name: mordred@inaugust.com - region_name: region-b.geo-1 + region_name: ca-ymq-1 + dns_api_version: 1 monty: profile: rax auth: @@ -316,7 +315,7 @@ Get a named cloud. import os_client_config cloud_config = os_client_config.OpenStackConfig().get_one_cloud( - 'hp', region_name='region-b.geo-1') + 'internap', region_name='ams01') print(cloud_config.name, cloud_config.region, cloud_config.config) Or, get all of the clouds. @@ -366,6 +365,14 @@ will get you a fully configured `novaclient` instance. nova = os_client_config.make_client('compute') +If you want to do the same thing but on a named cloud. + +.. code-block:: python + + import os_client_config + + nova = os_client_config.make_client('compute', cloud='mtvexx') + If you want to do the same thing but also support command line parsing. .. code-block:: python From cbab18b0fa87cb9ddbcd8e37aa4bfffd9b582b34 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 10 Feb 2016 16:05:15 -0800 Subject: [PATCH 253/365] Add osic vendor profile The new osic cloud is a thing. Add a vendor profile here to simplify using it. Change-Id: Iecd473c93cd1e1d8e2bf9a785f257a47df10351e --- doc/source/vendor-support.rst | 13 +++++++++++++ os_client_config/vendors/osic.json | 11 +++++++++++ 2 files changed, 24 insertions(+) create mode 100644 os_client_config/vendors/osic.json diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index e007b70f8..7b1cceb47 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -188,6 +188,19 @@ nyj01 New York, NY * Image API Version is 1 * Floating IPs are not supported +osic +---- + +https://cloud1.osic.org:5000 + +============== ================= +Region Name Human Name +============== ================= +RegionOne RegionOne +============== ================= + +* Public IPv4 is provided via NAT with Neutron Floating IP + ovh --- diff --git a/os_client_config/vendors/osic.json b/os_client_config/vendors/osic.json new file mode 100644 index 000000000..484d7111b --- /dev/null +++ b/os_client_config/vendors/osic.json @@ -0,0 +1,11 @@ +{ + "name": "osic", + "profile": { + "auth": { + "auth_url": "https://cloud1.osic.org:5000" + }, + "regions": [ + "RegionOne" + ] + } +} From a71511468ee7395040e8a0246b9cbba9b6de069f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 10 Mar 2016 16:17:08 -0500 Subject: [PATCH 254/365] Update reno for stable/mitaka Fix a few page titles at the same time Change-Id: I68d082f1cad51bbe58deed6a7e4b0de122c22fc7 --- releasenotes/source/index.rst | 7 ++++--- releasenotes/source/mitaka.rst | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 releasenotes/source/mitaka.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 386434ef8..2f4234a88 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -1,5 +1,6 @@ -Welcome to Nova Release Notes documentation! -============================================== +================================ + os-client-config Release Notes +================================ Contents ======== @@ -8,7 +9,7 @@ Contents :maxdepth: 2 unreleased - + mitaka Indices and tables ================== diff --git a/releasenotes/source/mitaka.rst b/releasenotes/source/mitaka.rst new file mode 100644 index 000000000..e54560965 --- /dev/null +++ b/releasenotes/source/mitaka.rst @@ -0,0 +1,6 @@ +=================================== + Mitaka Series Release Notes +=================================== + +.. release-notes:: + :branch: origin/stable/mitaka From 278a761df68d1e7d4d93ee2c6fb91f1a0e82e78a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 30 Mar 2016 16:10:04 -0700 Subject: [PATCH 255/365] Change network info indication to a generic list Networks can have more information than just internal or external. Notably, if you have two private networks and you're trying to assign floating ips, you need to know which network should be the recipient. This should be backwards compatible with existing external_network and internal_network options. Change-Id: I0d469339ba00486683fcd3ce2995002fa0a576d1 --- README.rst | 12 ++++--- os_client_config/config.py | 24 ++++++++++++++ os_client_config/tests/test_config.py | 31 +++++++++++++++++++ .../notes/network-list-e6e9dafdd8446263.yaml | 11 +++++++ 4 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/network-list-e6e9dafdd8446263.yaml diff --git a/README.rst b/README.rst index 2e584bd53..15f4bf09a 100644 --- a/README.rst +++ b/README.rst @@ -289,12 +289,16 @@ region. regions: - name: ams01 values: - external_network: inap-17037-WAN1654 - internal_network: inap-17037-LAN4820 + networks: + - name: inap-17037-WAN1654 + routes_externally: true + - name: inap-17037-LAN6745 - name: nyj01 values: - external_network: inap-17037-WAN7752 - internal_network: inap-17037-LAN6745 + networks: + - name: inap-17037-WAN1654 + routes_externally: true + - name: inap-17037-LAN6745 Usage ----- diff --git a/os_client_config/config.py b/os_client_config/config.py index 2f6adeb2b..98870f629 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -477,6 +477,7 @@ class OpenStackConfig(object): cloud = self._fix_backwards_auth_plugin(cloud) cloud = self._fix_backwards_project(cloud) cloud = self._fix_backwards_interface(cloud) + cloud = self._fix_backwards_networks(cloud) cloud = self._handle_domain_id(cloud) return cloud @@ -485,6 +486,29 @@ class OpenStackConfig(object): or 'project_id' in cloud['auth'] or 'project_name' in cloud['auth']) + def _fix_backwards_networks(self, cloud): + # Leave the external_network and internal_network keys in the + # dict because consuming code might be expecting them. + networks = cloud.get('networks', []) + for key in ('external_network', 'internal_network'): + external = key.startswith('external') + if key in cloud and 'networks' in cloud: + raise exceptions.OpenStackConfigException( + "Both {key} and networks were specified in the config." + " Please remove {key} from the config and use the network" + " list to configure network behavior.".format(key=key)) + if key in cloud: + warnings.warn( + "{key} is deprecated. Please replace with an entry in" + " a dict inside of the networks list with name: {name}" + " and routes_externally: {external}".format( + key=key, name=cloud[key], external=external)) + networks.append(dict( + name=cloud[key], + routes_externally=external)) + cloud['networks'] = networks + return cloud + def _handle_domain_id(self, cloud): # Allow people to just specify domain once if it's the same mappings = { diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 10a3d7b31..0db1457a9 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -753,3 +753,34 @@ class TestBackwardsCompatibility(base.TestCase): } } self.assertEqual(expected, result) + + def test_backwards_network_fail(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cloud = { + 'external_network': 'public', + 'networks': [ + {'name': 'private', 'routes_externally': False}, + ] + } + self.assertRaises( + exceptions.OpenStackConfigException, + c._fix_backwards_networks, cloud) + + def test_backwards_network(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cloud = { + 'external_network': 'public', + 'internal_network': 'private', + } + result = c._fix_backwards_networks(cloud) + expected = { + 'external_network': 'public', + 'internal_network': 'private', + 'networks': [ + {'name': 'public', 'routes_externally': True}, + {'name': 'private', 'routes_externally': False}, + ] + } + self.assertEqual(expected, result) diff --git a/releasenotes/notes/network-list-e6e9dafdd8446263.yaml b/releasenotes/notes/network-list-e6e9dafdd8446263.yaml new file mode 100644 index 000000000..83754883a --- /dev/null +++ b/releasenotes/notes/network-list-e6e9dafdd8446263.yaml @@ -0,0 +1,11 @@ +--- +features: + - Support added for configuring metadata about networks + for a cloud in a list of dicts, rather than in the + external_network and internal_network entries. The dicts + support a name and a routes_externally field, as well as + any other arbitrary metadata needed by consuming + applications. +deprecations: + - external_network and internal_network are deprecated and + should be replaced with the list of network dicts. From 7c439073f39010ad3ac937b8c9726da0f27976b7 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 2 Apr 2016 09:09:54 -0500 Subject: [PATCH 256/365] Flesh out netowrk config list Add support for indicating default_interface. Also, add some validation and normalization code, some interface methods and, shockingly, documentation. Change-Id: Ib45b68894585ac02821d5d2376510fd7a8e8ee40 --- README.rst | 2 +- doc/source/index.rst | 1 + doc/source/network-config.rst | 47 +++++++++++++++++++ os_client_config/cloud_config.py | 24 ++++++++++ os_client_config/config.py | 40 +++++++++++++++- os_client_config/tests/test_config.py | 36 +++++++++++++- .../notes/network-list-e6e9dafdd8446263.yaml | 5 +- 7 files changed, 147 insertions(+), 8 deletions(-) create mode 100644 doc/source/network-config.rst diff --git a/README.rst b/README.rst index 15f4bf09a..97a158429 100644 --- a/README.rst +++ b/README.rst @@ -277,7 +277,7 @@ To support this, the region list can actually be a list of dicts, and any setting that can be set at the cloud level can be overridden for that region. -:: +.. code-block:: yaml clouds: internap: diff --git a/doc/source/index.rst b/doc/source/index.rst index bf667b786..f7263c9c3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -6,6 +6,7 @@ vendor-support contributing installation + network-config api-reference releasenotes diff --git a/doc/source/network-config.rst b/doc/source/network-config.rst new file mode 100644 index 000000000..9bbbf9d7e --- /dev/null +++ b/doc/source/network-config.rst @@ -0,0 +1,47 @@ +============== +Network Config +============== + +There are several different qualities that networks in OpenStack might have +that might not be able to be automatically inferred from the available +metadata. To help users navigate more complex setups, `os-client-config` +allows configuring a list of network metadata. + +.. code-block:: yaml + + clouds: + amazing: + networks: + - name: blue + routes_externally: true + - name: purple + routes_externally: true + default_interface: true + - name: green + routes_externally: false + - name: purple + routes_externally: false + nat_destination: true + +Every entry must have a name field, which can hold either the name or the id +of the network. + +`routes_externally` is a boolean field that labels the network as handling +north/south traffic off of the cloud. In a public cloud this might be thought +of as the "public" network, but in private clouds it's possible it might +be an RFC1918 address. In either case, it's provides IPs to servers that +things not on the cloud can use. This value defaults to `false`, which +indicates only servers on the same network can talk to it. + +`default_interface` is a boolean field that indicates that the network is the +one that programs should use. It defaults to false. An example of needing to +use this value is a cloud with two private networks, and where a user is +running ansible in one of the servers to talk to other servers on the private +network. Because both networks are private, there would otherwise be no way +to determine which one should be used for the traffic. + +`nat_destination` is a boolean field that indicates which network floating +ips should be attached to. It defaults to false. Normally this can be inferred +by looking for a network that has subnets that have a gateway_ip. But it's +possible to have more than one network that satisfies that condition, so the +user might want to tell programs which one to pick. diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index b19607e03..e63bd127a 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -438,3 +438,27 @@ class CloudConfig(object): if resource not in expiration: return default return float(expiration[resource]) + + def get_external_networks(self): + """Get list of network names for external networks.""" + return [ + net['name'] for net in self._openstack_config['networks'] + if net['routes_externally']] + + def get_internal_networks(self): + """Get list of network names for internal networks.""" + return [ + net['name'] for net in self._openstack_config['networks'] + if not net['routes_externally']] + + def get_default_network(self): + """Get network used for default interactions.""" + for net in self._openstack_config['networks']: + if net['default_interface']: + return net + + def get_nat_destination(self): + """Get network used for NAT destination.""" + for net in self._openstack_config['networks']: + if net['nat_destination']: + return net diff --git a/os_client_config/config.py b/os_client_config/config.py index 98870f629..2dd49c3d1 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -83,6 +83,8 @@ def set_default(key, value): def get_boolean(value): + if value is None: + return False if type(value) is bool: return value if value.lower() == 'true': @@ -486,10 +488,37 @@ class OpenStackConfig(object): or 'project_id' in cloud['auth'] or 'project_name' in cloud['auth']) + def _validate_networks(self, networks, key): + value = None + for net in networks: + if value and net[key]: + raise exceptions.OpenStackConfigException( + "Duplicate network entries for {key}: {net1} and {net2}." + " Only one network can be flagged with {key}".format( + key=key, + net1=value['name'], + net2=net['name'])) + if not value and net[key]: + value = net + def _fix_backwards_networks(self, cloud): # Leave the external_network and internal_network keys in the # dict because consuming code might be expecting them. - networks = cloud.get('networks', []) + networks = [] + # Normalize existing network entries + for net in cloud.get('networks', []): + name = net.get('name') + if not name: + raise exceptions.OpenStackConfigException( + 'Entry in network list is missing required field "name".') + network = dict( + name=name, + routes_externally=get_boolean(net.get('routes_externally')), + nat_destination=get_boolean(net.get('nat_destination')), + default_interface=get_boolean(net.get('default_interface')), + ) + networks.append(network) + for key in ('external_network', 'internal_network'): external = key.startswith('external') if key in cloud and 'networks' in cloud: @@ -505,7 +534,14 @@ class OpenStackConfig(object): key=key, name=cloud[key], external=external)) networks.append(dict( name=cloud[key], - routes_externally=external)) + routes_externally=external, + nat_destination=not external, + default_interface=external)) + + # Validate that we don't have duplicates + self._validate_networks(networks, 'nat_destination') + self._validate_networks(networks, 'default_interface') + cloud['networks'] = networks return cloud diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 0db1457a9..7c2bec0f5 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -779,8 +779,40 @@ class TestBackwardsCompatibility(base.TestCase): 'external_network': 'public', 'internal_network': 'private', 'networks': [ - {'name': 'public', 'routes_externally': True}, - {'name': 'private', 'routes_externally': False}, + {'name': 'public', 'routes_externally': True, + 'nat_destination': False, 'default_interface': True}, + {'name': 'private', 'routes_externally': False, + 'nat_destination': True, 'default_interface': False}, ] } self.assertEqual(expected, result) + + def test_normalize_network(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cloud = { + 'networks': [ + {'name': 'private'} + ] + } + result = c._fix_backwards_networks(cloud) + expected = { + 'networks': [ + {'name': 'private', 'routes_externally': False, + 'nat_destination': False, 'default_interface': False}, + ] + } + self.assertEqual(expected, result) + + def test_single_default_interface(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cloud = { + 'networks': [ + {'name': 'blue', 'default_interface': True}, + {'name': 'purple', 'default_interface': True}, + ] + } + self.assertRaises( + exceptions.OpenStackConfigException, + c._fix_backwards_networks, cloud) diff --git a/releasenotes/notes/network-list-e6e9dafdd8446263.yaml b/releasenotes/notes/network-list-e6e9dafdd8446263.yaml index 83754883a..8f793c2bc 100644 --- a/releasenotes/notes/network-list-e6e9dafdd8446263.yaml +++ b/releasenotes/notes/network-list-e6e9dafdd8446263.yaml @@ -3,9 +3,8 @@ features: - Support added for configuring metadata about networks for a cloud in a list of dicts, rather than in the external_network and internal_network entries. The dicts - support a name and a routes_externally field, as well as - any other arbitrary metadata needed by consuming - applications. + support a name, a routes_externally field, a nat_destination + field and a default_interface field. deprecations: - external_network and internal_network are deprecated and should be replaced with the list of network dicts. From fdb80ad04f5611ebc62b4c64f896448c9213a75c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 2 Apr 2016 09:47:52 -0500 Subject: [PATCH 257/365] Clarify one-per-cloud network values Make it clear in the docs that default_interface and nat_destination can each be set only once per cloud. Change-Id: Ic862b9f4dc31580c4e192f13f100428bbec7faa2 --- doc/source/network-config.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/network-config.rst b/doc/source/network-config.rst index 9bbbf9d7e..5a27b8e0e 100644 --- a/doc/source/network-config.rst +++ b/doc/source/network-config.rst @@ -38,10 +38,12 @@ one that programs should use. It defaults to false. An example of needing to use this value is a cloud with two private networks, and where a user is running ansible in one of the servers to talk to other servers on the private network. Because both networks are private, there would otherwise be no way -to determine which one should be used for the traffic. +to determine which one should be used for the traffic. There can only be one +`default_interface` per cloud. `nat_destination` is a boolean field that indicates which network floating ips should be attached to. It defaults to false. Normally this can be inferred by looking for a network that has subnets that have a gateway_ip. But it's possible to have more than one network that satisfies that condition, so the -user might want to tell programs which one to pick. +user might want to tell programs which one to pick. There can be only one +`nat_destination` per cloud. From 5605034fc38862b19e6d2d96aa222dafc7762060 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 5 Apr 2016 10:38:46 -0400 Subject: [PATCH 258/365] Pull the network settings from the actual dict Turns out self._openstack_config is not the config dict. self.config is. Also, return names. Change-Id: Ib2013e737b506b3a2acd7aa7b7884240c25384c5 --- os_client_config/cloud_config.py | 14 ++++++++------ os_client_config/tests/base.py | 26 ++++++++++++++++++++++++++ os_client_config/tests/test_config.py | 21 +++++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index e63bd127a..5dfbba929 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -442,23 +442,25 @@ class CloudConfig(object): def get_external_networks(self): """Get list of network names for external networks.""" return [ - net['name'] for net in self._openstack_config['networks'] + net['name'] for net in self.config['networks'] if net['routes_externally']] def get_internal_networks(self): """Get list of network names for internal networks.""" return [ - net['name'] for net in self._openstack_config['networks'] + net['name'] for net in self.config['networks'] if not net['routes_externally']] def get_default_network(self): """Get network used for default interactions.""" - for net in self._openstack_config['networks']: + for net in self.config['networks']: if net['default_interface']: - return net + return net['name'] + return None def get_nat_destination(self): """Get network used for NAT destination.""" - for net in self._openstack_config['networks']: + for net in self.config['networks']: if net['nat_destination']: - return net + return net['name'] + return None diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 9b784b157..d046a941f 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -90,6 +90,32 @@ USER_CONF = { }, 'region_name': 'test-region', }, + '_test-cloud-networks_': { + 'auth': { + 'username': 'testuser', + 'password': 'testpass', + 'project_id': 12345, + 'auth_url': 'http://example.com/v2', + 'domain_id': '6789', + 'project_domain_id': '123456789', + }, + 'networks': [{ + 'name': 'a-public', + 'routes_externally': True, + }, { + 'name': 'another-public', + 'routes_externally': True, + 'default_interface': True, + }, { + 'name': 'a-private', + 'routes_externally': False, + }, { + 'name': 'another-private', + 'routes_externally': False, + 'nat_destination': True, + }], + 'region_name': 'test-region', + }, '_test_cloud_regions': { 'auth': { 'username': 'testuser', diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 7c2bec0f5..a0978f059 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -187,6 +187,26 @@ class TestConfig(base.TestCase): self.assertEqual('user', cc.auth['username']) self.assertEqual('testpass', cc.auth['password']) + def test_get_one_cloud_networks(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test-cloud-networks_') + self.assertEqual( + ['a-public', 'another-public'], cc.get_external_networks()) + self.assertEqual( + ['a-private', 'another-private'], cc.get_internal_networks()) + self.assertEqual('another-private', cc.get_nat_destination()) + self.assertEqual('another-public', cc.get_default_network()) + + def test_get_one_cloud_no_networks(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + cc = c.get_one_cloud('_test-cloud-domain-scoped_') + self.assertEqual([], cc.get_external_networks()) + self.assertEqual([], cc.get_internal_networks()) + self.assertIsNone(cc.get_nat_destination()) + self.assertIsNone(cc.get_default_network()) + def test_only_secure_yaml(self): c = config.OpenStackConfig(config_files=['nonexistent'], vendor_files=['nonexistent'], @@ -201,6 +221,7 @@ class TestConfig(base.TestCase): ['_test-cloud-domain-id_', '_test-cloud-domain-scoped_', '_test-cloud-int-project_', + '_test-cloud-networks_', '_test-cloud_', '_test-cloud_no_region', '_test_cloud_hyphenated', From d9f9c05bfb377f02b6150ca50030e088d2c19df5 Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Mon, 11 Apr 2016 16:05:27 +1000 Subject: [PATCH 259/365] Add version string Use PBR to add an __version__ string to os-client-config. Change-Id: I2293b2bd0dbbe0108e805be8ba02fbba9a5ab064 --- os_client_config/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index 6f78d2fca..be232fb68 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -14,10 +14,15 @@ import sys +import pbr.version + from os_client_config import cloud_config from os_client_config.config import OpenStackConfig # noqa +__version__ = pbr.version.VersionInfo('os_client_config').version_string() + + def get_config(service_key=None, options=None, **kwargs): config = OpenStackConfig() if options: From 1028f5ad7e21520f172c9c163e6b0ef66511bc13 Mon Sep 17 00:00:00 2001 From: Thomas Bechtold Date: Mon, 11 Apr 2016 10:30:24 +0200 Subject: [PATCH 260/365] Remove discover from test-requirements.txt discover is only needed for python 2.6 which is no longer supported. Change-Id: I8faeb05def94ac4adb2fe870fe141678dbd412ae --- test-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5e4c30446..0138f13f6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,7 +7,6 @@ hacking>=0.10.2,<0.11 # Apache-2.0 coverage>=3.6 extras fixtures>=0.3.14 -discover jsonschema>=2.0.0,<3.0.0,!=2.5.0 mock>=1.2 python-glanceclient>=0.18.0 From 700ab6f28220dbd09c2bf1c917245dd212c0fb75 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Fri, 22 Apr 2016 18:00:53 +0300 Subject: [PATCH 261/365] Fix formatting in readme file Change-Id: Ifa37d38b3c7689f703c7129459b15a367e2aafff --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 97a158429..ff95c07a8 100644 --- a/README.rst +++ b/README.rst @@ -399,6 +399,8 @@ in luck. The same interface for `make_client` is supported for `session_client` and will return you a keystoneauth Session object that is mounted on the endpoint for the service you're looking for. +.. code-block:: python + import os_client_config session = os_client_config.session_client('compute', cloud='vexxhost') From b0fa4383b06af6cef748e4a4f1fdcc39265915b9 Mon Sep 17 00:00:00 2001 From: "ChangBo Guo(gcb)" Date: Sat, 7 May 2016 16:56:32 +0800 Subject: [PATCH 262/365] drop python3.3 support in classifier We don't run python 3.3 CI jobs anymore, so just drop it from classifier. Change-Id: I871269ae54067aae40f5dc06affbd4354104ee91 --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 89df35c11..07ac625ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 [files] From 189a60475424e7efd41fe522d5a190f480ad1317 Mon Sep 17 00:00:00 2001 From: "ChangBo Guo(gcb)" Date: Sat, 7 May 2016 16:59:20 +0800 Subject: [PATCH 263/365] Trivial: remove openstack/common from flake8 exclude list openstack/common was used to kepp copied files from oslo-incubator, we don't use oslo-incubator stuff, so remove it. Change-Id: Id426c41e6ae277ed1f829820771d5ffc31a81166 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3df64e24c..d2aac05bd 100644 --- a/tox.ini +++ b/tox.ini @@ -34,5 +34,5 @@ commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasen [flake8] show-source = True builtins = _ -exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,releasenotes/source/conf.py +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,releasenotes/source/conf.py From 44efe9c955c0e74d47b374085d45130ac78e6a4c Mon Sep 17 00:00:00 2001 From: "ChangBo Guo(gcb)" Date: Sat, 7 May 2016 17:02:37 +0800 Subject: [PATCH 264/365] Trivial: Remove 'MANIFEST.in' Everything in this file is automatically generated by pbr. There appears to be no good reason to keep it around. Change-Id: I73eb120dedbdb6685862d26493fc178e6dee1353 --- MANIFEST.in | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 90f8a7aef..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,6 +0,0 @@ -include AUTHORS -include ChangeLog -exclude .gitignore -exclude .gitreview - -global-exclude *.pyc \ No newline at end of file From 090a265669940c98b84e610bbbdd10c29c3d3d02 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 9 May 2016 04:59:13 -0500 Subject: [PATCH 265/365] Workaround bad required params in troveclient troveclient requires username and password as parameters to the Client object, but if a Session is passed (like we do) that's not needed. A patch has been submitted to troveclient, but until that has been released, simply send None to both parameters. Change-Id: Ie130a4e83cceb7cab69bfbeb559493d195ef35e1 --- os_client_config/cloud_config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 5dfbba929..08c739a4d 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -351,6 +351,13 @@ class CloudConfig(object): constructor_kwargs['api_version'] = version else: constructor_kwargs['version'] = version + if service_key == 'database': + # TODO(mordred) Remove when https://review.openstack.org/314032 + # has landed and released. We're passing in a Session, but the + # trove Client object has username and password as required + # args + constructor_kwargs['username'] = None + constructor_kwargs['password'] = None return client_class(**constructor_kwargs) From fbe1b382dfeab55795341fd8c0709e9a5e8517bc Mon Sep 17 00:00:00 2001 From: Alvaro Lopez Garcia Date: Mon, 23 May 2016 11:29:23 +0200 Subject: [PATCH 266/365] Add missing "cloud" argument to _validate_auth_ksc The function _validate_auth_ksc was missing a "cloud" parameter so the exception formatting failed as it could not find that variable. Change-Id: Ia1caaa29fcb14d6ce7c16de1f78bbcae6c24adb0 --- os_client_config/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 2dd49c3d1..f1df7977b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -831,7 +831,7 @@ class OpenStackConfig(object): config['auth']['token'] = 'notused' return loading.get_plugin_loader(config['auth_type']) - def _validate_auth_ksc(self, config): + def _validate_auth_ksc(self, config, cloud): try: import keystoneclient.auth as ksc_auth except ImportError: @@ -1005,7 +1005,7 @@ class OpenStackConfig(object): self.log.debug("Deferring keystone exception: {e}".format(e=e)) auth_plugin = None try: - config = self._validate_auth_ksc(config) + config = self._validate_auth_ksc(config, cloud) except Exception: raise e else: From 41ac1562b5a10f7dcbdd4131b56784763f40eb69 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 30 May 2016 18:33:38 -0400 Subject: [PATCH 267/365] Add helper method for OpenStack SDK constructor openstacksdk already has a helper method for dealing with occ, but if a user is already using the occ helper methods, there is no reason we should not provide them an easy path to using the SDK. Change-Id: I1040efb94385fdac0aa02ac960ba95089b954377 --- README.rst | 37 +++++++++++++++++++ os_client_config/__init__.py | 13 +++++++ .../notes/sdk-helper-41f8d815cfbcfb00.yaml | 4 ++ 3 files changed, 54 insertions(+) create mode 100644 releasenotes/notes/sdk-helper-41f8d815cfbcfb00.yaml diff --git a/README.rst b/README.rst index ff95c07a8..223fcfa20 100644 --- a/README.rst +++ b/README.rst @@ -355,6 +355,43 @@ with - as well as a consumption argument. cloud = cloud_config.get_one_cloud(argparse=options) +Constructing OpenStack SDK object +--------------------------------- + +If what you want to do is get an OpenStack SDK Connection and you want it to +do all the normal things related to clouds.yaml, `OS_` environment variables, +a helper function is provided. The following will get you a fully configured +`openstacksdk` instance. + +.. code-block:: python + + import os_client_config + + sdk = os_client_config.make_sdk() + +If you want to do the same thing but on a named cloud. + +.. code-block:: python + + import os_client_config + + sdk = os_client_config.make_sdk(cloud='mtvexx') + +If you want to do the same thing but also support command line parsing. + +.. code-block:: python + + import argparse + + import os_client_config + + sdk = os_client_config.make_sdk(options=argparse.ArgumentParser()) + +It should be noted that OpenStack SDK has ways to construct itself that allow +for additional flexibility. If the helper function here does not meet your +needs, you should see the `from_config` method of +`openstack.connection.Connection `_ + Constructing Legacy Client objects ---------------------------------- diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index be232fb68..c88ccb2c6 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -67,3 +67,16 @@ def make_client(service_key, constructor=None, options=None, **kwargs): if not constructor: constructor = cloud_config._get_client(service_key) return cloud.get_legacy_client(service_key, constructor) + + +def make_sdk(options=None, **kwargs): + """Simple wrapper for getting an OpenStack SDK Connection. + + For completeness, provide a mechanism that matches make_client and + session_client. The heavy lifting here is done in openstacksdk. + + :rtype: :class:`~openstack.connection.Connection` + """ + from openstack import connection + cloud = get_config(options=options, **kwargs) + return connection.from_config(cloud_config=cloud, options=options) diff --git a/releasenotes/notes/sdk-helper-41f8d815cfbcfb00.yaml b/releasenotes/notes/sdk-helper-41f8d815cfbcfb00.yaml new file mode 100644 index 000000000..a18b57dc3 --- /dev/null +++ b/releasenotes/notes/sdk-helper-41f8d815cfbcfb00.yaml @@ -0,0 +1,4 @@ +--- +features: + - Added helper method for constructing OpenStack SDK + Connection objects. From 6a834063a25852f7f6cd45d6f7331aa0f77ff4c5 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 1 Jun 2016 10:28:51 +0300 Subject: [PATCH 268/365] Rename session_client to make_rest_client While writing some docs, it became clear that session_client was just a horrible horrible name and that I'm a bad person. Rename it so that we can make docs that make humans happy. Also, move the REST client section of the README up a bit. Change-Id: I1a27853e3031489da5916308a76f19bc72185d24 --- README.rst | 34 +++++++++---------- os_client_config/__init__.py | 8 +++-- .../make-rest-client-dd3d365632a26fa0.yaml | 4 +++ 3 files changed, 26 insertions(+), 20 deletions(-) create mode 100644 releasenotes/notes/make-rest-client-dd3d365632a26fa0.yaml diff --git a/README.rst b/README.rst index 223fcfa20..99ed287b9 100644 --- a/README.rst +++ b/README.rst @@ -392,6 +392,23 @@ for additional flexibility. If the helper function here does not meet your needs, you should see the `from_config` method of `openstack.connection.Connection `_ +Constructing REST API Clients +----------------------------- + +What if you want to make direct REST calls via a Session interface? You're +in luck. The same interface for `make_sdk` is supported for +`make_rest_client` and will return you a keystoneauth Session object that is +mounted on the endpoint for the service you're looking for. + +.. code-block:: python + + import os_client_config + + session = os_client_config.make_rest_client('compute', cloud='vexxhost') + + response = session.get('/servers') + server_list = response.json()['servers'] + Constructing Legacy Client objects ---------------------------------- @@ -428,23 +445,6 @@ If you want to do the same thing but also support command line parsing. If you want to get fancier than that in your python, then the rest of the API is available to you. But often times, you just want to do the one thing. -Constructing Mounted Session Objects ------------------------------------- - -What if you want to make direct REST calls via a Session interface? You're -in luck. The same interface for `make_client` is supported for `session_client` -and will return you a keystoneauth Session object that is mounted on the -endpoint for the service you're looking for. - -.. code-block:: python - - import os_client_config - - session = os_client_config.session_client('compute', cloud='vexxhost') - - response = session.get('/servers') - server_list = response.json()['servers'] - Source ------ diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index c88ccb2c6..6142853ef 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -34,7 +34,7 @@ def get_config(service_key=None, options=None, **kwargs): return config.get_one_cloud(options=parsed_options, **kwargs) -def session_client(service_key, options=None, **kwargs): +def make_rest_client(service_key, options=None, **kwargs): """Simple wrapper function. It has almost no features. This will get you a raw requests Session Adapter that is mounted @@ -50,7 +50,9 @@ def session_client(service_key, options=None, **kwargs): cloud = get_config(service_key=service_key, options=options, **kwargs) return cloud.get_session_client(service_key) # Backwards compat - simple_client was a terrible name -simple_client = session_client +simple_client = make_rest_client +# Backwards compat - session_client was a terrible name +session_client = make_rest_client def make_client(service_key, constructor=None, options=None, **kwargs): @@ -73,7 +75,7 @@ def make_sdk(options=None, **kwargs): """Simple wrapper for getting an OpenStack SDK Connection. For completeness, provide a mechanism that matches make_client and - session_client. The heavy lifting here is done in openstacksdk. + make_rest_client. The heavy lifting here is done in openstacksdk. :rtype: :class:`~openstack.connection.Connection` """ diff --git a/releasenotes/notes/make-rest-client-dd3d365632a26fa0.yaml b/releasenotes/notes/make-rest-client-dd3d365632a26fa0.yaml new file mode 100644 index 000000000..8e34e5198 --- /dev/null +++ b/releasenotes/notes/make-rest-client-dd3d365632a26fa0.yaml @@ -0,0 +1,4 @@ +--- +deprecations: + - Renamed session_client to make_rest_client. session_client + will continue to be supported for backwards compatability. From 7d63f12eddb250542587cab8e4a2ca9088ec0fbb Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 1 Jun 2016 10:37:58 +0300 Subject: [PATCH 269/365] Add shade constructor helper method We have helper factory methods for REST Client, legacy client and OpenStack SDK all with the same interface ... we might as well have one for shade too. It makes documenting and talking about the simple case of all of them easy. Change-Id: I046da85ae4a3e2a6333223921d5ae9ce3673121d --- README.rst | 33 +++++++++++++++++++ os_client_config/__init__.py | 12 +++++++ .../notes/shade-helper-568f8cb372eef6d9.yaml | 4 +++ 3 files changed, 49 insertions(+) create mode 100644 releasenotes/notes/shade-helper-568f8cb372eef6d9.yaml diff --git a/README.rst b/README.rst index 99ed287b9..3d97476b1 100644 --- a/README.rst +++ b/README.rst @@ -392,6 +392,39 @@ for additional flexibility. If the helper function here does not meet your needs, you should see the `from_config` method of `openstack.connection.Connection `_ +Constructing shade objects +-------------------------- + +If what you want to do is get a +`shade `_ OpenStackCloud object, a +helper function that honors clouds.yaml and `OS_` environment variables is +provided. The following will get you a fully configured `OpenStackCloud` +instance. + +.. code-block:: python + + import os_client_config + + cloud = os_client_config.make_shade() + +If you want to do the same thing but on a named cloud. + +.. code-block:: python + + import os_client_config + + cloud = os_client_config.make_shade(cloud='mtvexx') + +If you want to do the same thing but also support command line parsing. + +.. code-block:: python + + import argparse + + import os_client_config + + cloud = os_client_config.make_shade(options=argparse.ArgumentParser()) + Constructing REST API Clients ----------------------------- diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index 6142853ef..09d74423b 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -82,3 +82,15 @@ def make_sdk(options=None, **kwargs): from openstack import connection cloud = get_config(options=options, **kwargs) return connection.from_config(cloud_config=cloud, options=options) + + +def make_shade(options=None, **kwargs): + """Simple wrapper for getting a Shade OpenStackCloud object + + A mechanism that matches make_sdk, make_client and make_rest_client. + + :rtype: :class:`~shade.OpenStackCloud` + """ + import shade + cloud = get_config(options=options, **kwargs) + return shade.OpenStackCloud(cloud_config=cloud, **kwargs) diff --git a/releasenotes/notes/shade-helper-568f8cb372eef6d9.yaml b/releasenotes/notes/shade-helper-568f8cb372eef6d9.yaml new file mode 100644 index 000000000..70aab0a13 --- /dev/null +++ b/releasenotes/notes/shade-helper-568f8cb372eef6d9.yaml @@ -0,0 +1,4 @@ +--- +features: + - Added helper method for constructing shade + OpenStackCloud objects. From 4f36eca18f30953352e3fea42776e2e49d1db5fe Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 1 Jun 2016 10:51:44 +0300 Subject: [PATCH 270/365] Reword the entries in the README a bit Wanted to make each section a little better, but also to start to indicate that legacy clients should really not be your first choice. Change-Id: I26e08d037c7b28ced22a2a0126693d7e3e779f58 --- README.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 3d97476b1..a47e98bed 100644 --- a/README.rst +++ b/README.rst @@ -429,9 +429,10 @@ Constructing REST API Clients ----------------------------- What if you want to make direct REST calls via a Session interface? You're -in luck. The same interface for `make_sdk` is supported for -`make_rest_client` and will return you a keystoneauth Session object that is -mounted on the endpoint for the service you're looking for. +in luck. A similar interface is available as with `openstacksdk` and `shade`. +The main difference is that you need to specify which service you want to +talk to and `make_rest_client` will return you a keystoneauth Session object +that is mounted on the endpoint for the service you're looking for. .. code-block:: python @@ -445,9 +446,9 @@ mounted on the endpoint for the service you're looking for. Constructing Legacy Client objects ---------------------------------- -If all you want to do is get a Client object from a python-\*client library, +If you want get an old-style Client object from a python-\*client library, and you want it to do all the normal things related to clouds.yaml, `OS_` -environment variables, a helper function is provided. The following +environment variables, a helper function is also provided. The following will get you a fully configured `novaclient` instance. .. code-block:: python From 891fa1c60fc948aac158ea939c5a10bd1cab04d9 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Sat, 18 Jun 2016 08:17:03 -0500 Subject: [PATCH 271/365] Refactor fix magic in get_one_cloud() Extract the magic argument fixes from get_ne_cloud() so they can be extended or modified. Needed by OSC in order to maintain compatibility due to the much earlier loading of auth plugins than before. Have I mentioned that really fouled things up for OSC? Change-Id: I22cd890f9cbd605dcd615f82b3e65c58f52ff114 --- os_client_config/config.py | 71 ++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index f1df7977b..dc9b73172 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -475,14 +475,6 @@ class OpenStackConfig(object): " the cloud '{1}'".format(profile_name, name)) - def _fix_backwards_madness(self, cloud): - cloud = self._fix_backwards_auth_plugin(cloud) - cloud = self._fix_backwards_project(cloud) - cloud = self._fix_backwards_interface(cloud) - cloud = self._fix_backwards_networks(cloud) - cloud = self._handle_domain_id(cloud) - return cloud - def _project_scoped(self, cloud): return ('project_id' in cloud or 'project_name' in cloud or 'project_id' in cloud['auth'] @@ -812,7 +804,7 @@ class OpenStackConfig(object): def auth_config_hook(self, config): """Allow examination of config values before loading auth plugin - OpenStackClient will override this to perform additional chacks + OpenStackClient will override this to perform additional checks on auth_type. """ return config @@ -911,6 +903,41 @@ class OpenStackConfig(object): return config + def magic_fixes(self, config): + """Perform the set of magic argument fixups""" + + # Infer token plugin if a token was given + if (('auth' in config and 'token' in config['auth']) or + ('auth_token' in config and config['auth_token']) or + ('token' in config and config['token'])): + config.setdefault('token', config.pop('auth_token', None)) + + # These backwards compat values are only set via argparse. If it's + # there, it's because it was passed in explicitly, and should win + config = self._fix_backwards_api_timeout(config) + if 'endpoint_type' in config: + config['interface'] = config.pop('endpoint_type') + + config = self._fix_backwards_auth_plugin(config) + config = self._fix_backwards_project(config) + config = self._fix_backwards_interface(config) + config = self._fix_backwards_networks(config) + config = self._handle_domain_id(config) + + for key in BOOL_KEYS: + if key in config: + if type(config[key]) is not bool: + config[key] = get_boolean(config[key]) + + # TODO(mordred): Special casing auth_url here. We should + # come back to this betterer later so that it's + # more generalized + if 'auth' in config and 'auth_url' in config['auth']: + config['auth']['auth_url'] = config['auth']['auth_url'].format( + **config) + + return config + def get_one_cloud(self, cloud=None, validate=True, argparse=None, **kwargs): """Retrieve a single cloud configuration and merge additional options @@ -961,31 +988,7 @@ class OpenStackConfig(object): else: config[key] = val - # Infer token plugin if a token was given - if (('auth' in config and 'token' in config['auth']) or - ('auth_token' in config and config['auth_token']) or - ('token' in config and config['token'])): - config.setdefault('token', config.pop('auth_token', None)) - - # These backwards compat values are only set via argparse. If it's - # there, it's because it was passed in explicitly, and should win - config = self._fix_backwards_api_timeout(config) - if 'endpoint_type' in config: - config['interface'] = config.pop('endpoint_type') - - config = self._fix_backwards_madness(config) - - for key in BOOL_KEYS: - if key in config: - if type(config[key]) is not bool: - config[key] = get_boolean(config[key]) - - # TODO(mordred): Special casing auth_url here. We should - # come back to this betterer later so that it's - # more generalized - if 'auth' in config and 'auth_url' in config['auth']: - config['auth']['auth_url'] = config['auth']['auth_url'].format( - **config) + config = self.magic_fixes(config) # NOTE(dtroyer): OSC needs a hook into the auth args before the # plugin is loaded in order to maintain backward- From 481be16b8b385e1fcccd34607da8a8a3f5bde69f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 Jul 2016 17:34:11 +0900 Subject: [PATCH 272/365] Add support for deprecating cloud profiles We've had HP shut down their public cloud, and DreamHost has recent spun up a new cloud with a different operational profile and auth_url that is essentially a replacement cloud. That means people using the old information need to do things, so we need to be able to communicate that to them. Add support for adding a deprecated status to a vendor profile, as well as a verbose message explaining what the user may do to remediate. Change-Id: I19b67d7cd71fba2d9da0e3a6adb2d229ead65396 --- os_client_config/config.py | 6 ++++++ os_client_config/defaults.json | 2 ++ os_client_config/vendor-schema.json | 11 +++++++++++ 3 files changed, 19 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index f1df7977b..84476b2ba 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -468,6 +468,12 @@ class OpenStackConfig(object): else: profile_data = vendors.get_profile(profile_name) if profile_data: + status = profile_data.pop('status', 'active') + message = profile_data.pop('message', '') + if status == 'deprecated': + warnings.warn( + "{profile_name} is deprecated: {message}".format( + profile_name=profile_name, message=message)) _auth_update(cloud, profile_data) else: # Can't find the requested vendor config, go about business diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index f501862d5..ba8bf3923 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -13,10 +13,12 @@ "image_api_version": "2", "image_format": "qcow2", "key_manager_api_version": "v1", + "message": "", "metering_api_version": "2", "network_api_version": "2", "object_store_api_version": "1", "orchestration_api_version": "1", "secgroup_source": "neutron", + "status": "active", "volume_api_version": "2" } diff --git a/os_client_config/vendor-schema.json b/os_client_config/vendor-schema.json index 6c57ba4bc..a5bee27f6 100644 --- a/os_client_config/vendor-schema.json +++ b/os_client_config/vendor-schema.json @@ -55,12 +55,23 @@ "default": "public", "enum": [ "public", "internal", "admin" ] }, + "message": { + "name": "Status message", + "description": "Optional message with information related to status", + "type": "string" + }, "secgroup_source": { "name": "Security Group Source", "description": "Which service provides security groups", "enum": [ "neutron", "nova", "None" ], "default": "neutron" }, + "status": { + "name": "Vendor status", + "description": "Status of the vendor's cloud", + "enum": [ "active", "deprecated"], + "default": "active" + }, "compute_service_name": { "name": "Compute API Service Name", "description": "Compute API Service Name", From d9e9bb791ba169e828d6068534ee2f430f4668eb Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 14 Jul 2016 10:30:49 +0800 Subject: [PATCH 273/365] Add support for listing a cloud as shut down We've had one vendor cloud go away in the past, and there is one existing deprecated cloud currently. Add support for providing the user with an informative error message in the case where they attempt to use such a cloud. Change-Id: I894e0c0a4786e60fce1238bb2883828e89d44b01 --- os_client_config/config.py | 5 +++++ os_client_config/vendor-schema.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 84476b2ba..f9b2de560 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -474,6 +474,11 @@ class OpenStackConfig(object): warnings.warn( "{profile_name} is deprecated: {message}".format( profile_name=profile_name, message=message)) + elif status == 'shutdown': + raise exceptions.OpenStackConfigException( + "{profile_name} references a cloud that no longer" + " exists: {message}".format( + profile_name=profile_name, message=message)) _auth_update(cloud, profile_data) else: # Can't find the requested vendor config, go about business diff --git a/os_client_config/vendor-schema.json b/os_client_config/vendor-schema.json index a5bee27f6..6a6f4561e 100644 --- a/os_client_config/vendor-schema.json +++ b/os_client_config/vendor-schema.json @@ -69,7 +69,7 @@ "status": { "name": "Vendor status", "description": "Status of the vendor's cloud", - "enum": [ "active", "deprecated"], + "enum": [ "active", "deprecated", "shutdown"], "default": "active" }, "compute_service_name": { From 1f7ecbc3ff482de3ae8323fa5a62a359510e13d9 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 Jul 2016 18:23:02 +0900 Subject: [PATCH 274/365] Update citycloud to list new regions Frankfurt, Buffalo and Los Angeles - oh my! Change-Id: I17d6f46de2a9af82f221b971a359d53eb471f8fa --- doc/source/vendor-support.rst | 5 ++++- os_client_config/vendors/citycloud.json | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index f97c9f6ff..1053b3d4b 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -62,9 +62,12 @@ https://identity1.citycloud.com:5000/v3/ ============== ================ Region Name Human Name ============== ================ +Buf1 Buffalo, NY +Fra1 Frankfurt, DE +Kna1 Karlskrona, SE +La1 Los Angeles, CA Lon1 London, UK Sto2 Stockholm, SE -Kna1 Karlskrona, SE ============== ================ * Identity API Version is 3 diff --git a/os_client_config/vendors/citycloud.json b/os_client_config/vendors/citycloud.json index 64cadce9a..097ddfdb3 100644 --- a/os_client_config/vendors/citycloud.json +++ b/os_client_config/vendors/citycloud.json @@ -5,6 +5,9 @@ "auth_url": "https://identity1.citycloud.com:5000/v3/" }, "regions": [ + "Buf1", + "La1", + "Fra1", "Lon1", "Sto2", "Kna1" From 05b3c933b34e9cec9eb859a15392862918b3eb5f Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Tue, 2 Aug 2016 21:15:49 -0500 Subject: [PATCH 275/365] Fix precedence for pass-in options The passed-in argparse Namespace is flat and does not have an 'auth' dict, all of the auth options are in the top level. If the loaded dict from clouds.yaml as an 'auth' dict, that used to totally override any passed-in options. This is backwards. Make the passed-in auth options always win over clouds.yaml as this is the only way to change/override those values from the command line. Change-Id: Ic2752a396d45deeda19a16389437679829e0844d --- os_client_config/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index f1df7977b..0f19dd187 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -885,14 +885,14 @@ class OpenStackConfig(object): plugin_options = loader.get_options() for p_opt in plugin_options: - # if it's in config.auth, win, kill it from config dict - # if it's in config and not in config.auth, move it + # if it's in config, win, move it and kill it from config dict + # if it's in config.auth but not in config we're good # deprecated loses to current # provided beats default, deprecated or not - winning_value = self._find_winning_auth_value( - p_opt, config['auth']) + winning_value = self._find_winning_auth_value(p_opt, config) if not winning_value: - winning_value = self._find_winning_auth_value(p_opt, config) + winning_value = self._find_winning_auth_value( + p_opt, config['auth']) # Clean up after ourselves for opt in [p_opt.name] + [o.name for o in p_opt.deprecated]: From 9c699ed3a649d7d6d1f3c9d8a13eda483f948512 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 Jul 2016 17:36:48 +0900 Subject: [PATCH 276/365] Add the new DreamCompute cloud It does things the happy way - direct routing of IPv4 and IPv6. However, it's a respin, so we're naming it dreamcompute rather than dreamhost so that users don't get broken by the shift. Remove dreamhost from the docs - people looking at documentation should be using the new region. Change-Id: I92eb38635c4389d2e9326fab038137a673497fa8 --- doc/source/vendor-support.rst | 17 +++++++++++++++++ os_client_config/vendors/dreamcompute.json | 11 +++++++++++ os_client_config/vendors/dreamhost.json | 2 ++ 3 files changed, 30 insertions(+) create mode 100644 os_client_config/vendors/dreamcompute.json diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index f97c9f6ff..0297b7d24 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -99,9 +99,26 @@ sal01 Manchester, UK * Image API Version is 1 +dreamcompute +------------ + +https://iad2.dream.io:5000 + +============== ================ +Region Name Human Name +============== ================ +RegionOne Region One +============== ================ + +* Identity API Version is 3 +* Images must be in `raw` format +* IPv6 is provided to every server + dreamhost --------- +Deprecated, please use dreamcompute + https://keystone.dream.io/v2.0 ============== ================ diff --git a/os_client_config/vendors/dreamcompute.json b/os_client_config/vendors/dreamcompute.json new file mode 100644 index 000000000..8244cf77c --- /dev/null +++ b/os_client_config/vendors/dreamcompute.json @@ -0,0 +1,11 @@ +{ + "name": "dreamcompute", + "profile": { + "auth": { + "auth_url": "https://iad2.dream.io:5000" + }, + "identity_api_version": "3", + "region_name": "RegionOne", + "image_format": "raw" + } +} diff --git a/os_client_config/vendors/dreamhost.json b/os_client_config/vendors/dreamhost.json index 6fc2ccf8a..ea2ebac1e 100644 --- a/os_client_config/vendors/dreamhost.json +++ b/os_client_config/vendors/dreamhost.json @@ -1,6 +1,8 @@ { "name": "dreamhost", "profile": { + "status": "deprecated", + "message": "The dreamhost profile is deprecated. Please use the dreamcompute profile instead", "auth": { "auth_url": "https://keystone.dream.io" }, From 37dcc7e8d874a2e5325d7e473d883c373bf694ca Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Wed, 3 Aug 2016 10:43:15 -0500 Subject: [PATCH 277/365] Add release notes for 1.19.0 release Change-Id: I92ffcf611d31f7a4f11e5228022ea64864823389 --- .../notes/cloud-profile-status-e0d29b5e2f10e95c.yaml | 6 ++++++ releasenotes/notes/magic-fixes-dca4ae4dac2441a8.yaml | 6 ++++++ releasenotes/notes/option-precedence-1fecab21fdfb2c33.yaml | 7 +++++++ releasenotes/notes/vendor-updates-f11184ba56bb27cf.yaml | 4 ++++ 4 files changed, 23 insertions(+) create mode 100644 releasenotes/notes/cloud-profile-status-e0d29b5e2f10e95c.yaml create mode 100644 releasenotes/notes/magic-fixes-dca4ae4dac2441a8.yaml create mode 100644 releasenotes/notes/option-precedence-1fecab21fdfb2c33.yaml create mode 100644 releasenotes/notes/vendor-updates-f11184ba56bb27cf.yaml diff --git a/releasenotes/notes/cloud-profile-status-e0d29b5e2f10e95c.yaml b/releasenotes/notes/cloud-profile-status-e0d29b5e2f10e95c.yaml new file mode 100644 index 000000000..b447ed0a4 --- /dev/null +++ b/releasenotes/notes/cloud-profile-status-e0d29b5e2f10e95c.yaml @@ -0,0 +1,6 @@ +--- +features: + - Add a field to vendor cloud profiles to indicate + active, deprecated and shutdown status. A message to + the user is triggered when attempting to use cloud + with either deprecated or shutdown status. diff --git a/releasenotes/notes/magic-fixes-dca4ae4dac2441a8.yaml b/releasenotes/notes/magic-fixes-dca4ae4dac2441a8.yaml new file mode 100644 index 000000000..570e4dcca --- /dev/null +++ b/releasenotes/notes/magic-fixes-dca4ae4dac2441a8.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - Refactor ``OpenStackConfig._fix_backward_madness()`` into + ``OpenStackConfig.magic_fixes()`` that allows subclasses + to inject more fixup magic into the flow during + ``get_one_cloud()`` processing. diff --git a/releasenotes/notes/option-precedence-1fecab21fdfb2c33.yaml b/releasenotes/notes/option-precedence-1fecab21fdfb2c33.yaml new file mode 100644 index 000000000..06e6bd2f6 --- /dev/null +++ b/releasenotes/notes/option-precedence-1fecab21fdfb2c33.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - Reverse the order of option selction in + ``OpenStackConfig._validate_auth()`` to prefer auth options + passed in (from argparse) over those found in clouds.yaml. + This allows the application to override config profile + auth settings. diff --git a/releasenotes/notes/vendor-updates-f11184ba56bb27cf.yaml b/releasenotes/notes/vendor-updates-f11184ba56bb27cf.yaml new file mode 100644 index 000000000..e1d6d41a2 --- /dev/null +++ b/releasenotes/notes/vendor-updates-f11184ba56bb27cf.yaml @@ -0,0 +1,4 @@ +--- +other: + - Add citycloud regions for Buffalo, Frankfurt, Karlskrona and Los Angles + - Add new DreamCompute cloud and deprecate DreamHost cloud From d71a90220397f1baa80e943a37854a5be2c8726c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 4 Aug 2016 20:45:49 +0000 Subject: [PATCH 278/365] Revert "Fix precedence for pass-in options" This reverts commit 05b3c933b34e9cec9eb859a15392862918b3eb5f. Change-Id: I07dd701ca911cd12701519d2e6d624c69baa0848 --- os_client_config/config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 0f19dd187..f1df7977b 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -885,14 +885,14 @@ class OpenStackConfig(object): plugin_options = loader.get_options() for p_opt in plugin_options: - # if it's in config, win, move it and kill it from config dict - # if it's in config.auth but not in config we're good + # if it's in config.auth, win, kill it from config dict + # if it's in config and not in config.auth, move it # deprecated loses to current # provided beats default, deprecated or not - winning_value = self._find_winning_auth_value(p_opt, config) + winning_value = self._find_winning_auth_value( + p_opt, config['auth']) if not winning_value: - winning_value = self._find_winning_auth_value( - p_opt, config['auth']) + winning_value = self._find_winning_auth_value(p_opt, config) # Clean up after ourselves for opt in [p_opt.name] + [o.name for o in p_opt.deprecated]: From ddfed7f2fbabd1eba3b6ac700cee065c070433a7 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 4 Aug 2016 16:18:42 -0500 Subject: [PATCH 279/365] Pass the argparse data into to validate_auth We have two contradictory precedence needs that are impossible to satisfy because we're losing knowledge of the source of data as we build the ultimate dict of data. If we carry the argparse data along in a separate bucket for longer, we can check to see if it's there so that it can win, but so that in situations where kwargs is complex and contains both a top-level and an auth dict we don't assume that the values that are not in the auth dict came from argparse. By doing this, we can establish that precedence for auth args is: - argparse - auth dict - top level kwargs Change-Id: I9eca5937077f5873f7896b6745951fb8d8c4747c --- os_client_config/config.py | 60 +++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index f1df7977b..2aa053034 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -763,7 +763,7 @@ class OpenStackConfig(object): cloud, region_name=region['name'])) return clouds - def _fix_args(self, args, argparse=None): + def _fix_args(self, args=None, argparse=None): """Massage the passed-in options Replace - with _ and strip os_ prefixes. @@ -771,6 +771,8 @@ class OpenStackConfig(object): Convert an argparse Namespace object to a dict, removing values that are either None or ''. """ + if not args: + args = {} if argparse: # Convert the passed-in Namespace @@ -831,7 +833,7 @@ class OpenStackConfig(object): config['auth']['token'] = 'notused' return loading.get_plugin_loader(config['auth_type']) - def _validate_auth_ksc(self, config, cloud): + def _validate_auth_ksc(self, config, cloud, fixed_argparse): try: import keystoneclient.auth as ksc_auth except ImportError: @@ -842,14 +844,22 @@ class OpenStackConfig(object): config['auth_type']).get_options() for p_opt in plugin_options: + # if it's in argparse, it was passed on the command line and wins # if it's in config.auth, win, kill it from config dict # if it's in config and not in config.auth, move it # deprecated loses to current # provided beats default, deprecated or not winning_value = self._find_winning_auth_value( - p_opt, config['auth']) - if not winning_value: - winning_value = self._find_winning_auth_value(p_opt, config) + p_opt, fixed_argparse) + if winning_value: + found_in_argparse = True + else: + found_in_argparse = False + winning_value = self._find_winning_auth_value( + p_opt, config['auth']) + if not winning_value: + winning_value = self._find_winning_auth_value( + p_opt, config) # if the plugin tells us that this value is required # then error if it's doesn't exist now @@ -865,7 +875,12 @@ class OpenStackConfig(object): # Clean up after ourselves for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: opt = opt.replace('-', '_') - config.pop(opt, None) + # don't do this if the value came from argparse, because we + # don't (yet) know if the value in not-auth came from argparse + # overlay or from someone passing in a dict to kwargs + # TODO(mordred) Fix that data path too + if not found_in_argparse: + config.pop(opt, None) config['auth'].pop(opt, None) if winning_value: @@ -879,25 +894,38 @@ class OpenStackConfig(object): return config - def _validate_auth(self, config, loader): + def _validate_auth(self, config, loader, fixed_argparse): # May throw a keystoneauth1.exceptions.NoMatchingPlugin plugin_options = loader.get_options() for p_opt in plugin_options: + # if it's in argparse, it was passed on the command line and wins # if it's in config.auth, win, kill it from config dict # if it's in config and not in config.auth, move it # deprecated loses to current # provided beats default, deprecated or not winning_value = self._find_winning_auth_value( - p_opt, config['auth']) - if not winning_value: - winning_value = self._find_winning_auth_value(p_opt, config) + p_opt, fixed_argparse) + if winning_value: + found_in_argparse = True + else: + found_in_argparse = False + winning_value = self._find_winning_auth_value( + p_opt, config['auth']) + if not winning_value: + winning_value = self._find_winning_auth_value( + p_opt, config) # Clean up after ourselves for opt in [p_opt.name] + [o.name for o in p_opt.deprecated]: opt = opt.replace('-', '_') - config.pop(opt, None) + # don't do this if the value came from argparse, because we + # don't (yet) know if the value in not-auth came from argparse + # overlay or from someone passing in a dict to kwargs + # TODO(mordred) Fix that data path too + if not found_in_argparse: + config.pop(opt, None) config['auth'].pop(opt, None) if winning_value: @@ -932,6 +960,11 @@ class OpenStackConfig(object): """ args = self._fix_args(kwargs, argparse=argparse) + # Run the fix just for argparse by itself. We need to + # have a copy of the argparse options separately from + # any merged copied later in validate_auth so that we + # can determine precedence + fixed_argparse = self._fix_args(argparse=argparse) if cloud is None: if 'cloud' in args: @@ -995,7 +1028,7 @@ class OpenStackConfig(object): if validate: try: loader = self._get_auth_loader(config) - config = self._validate_auth(config, loader) + config = self._validate_auth(config, loader, fixed_argparse) auth_plugin = loader.load_from_options(**config['auth']) except Exception as e: # We WANT the ksa exception normally @@ -1005,7 +1038,8 @@ class OpenStackConfig(object): self.log.debug("Deferring keystone exception: {e}".format(e=e)) auth_plugin = None try: - config = self._validate_auth_ksc(config, cloud) + config = self._validate_auth_ksc( + config, cloud, fixed_argparse) except Exception: raise e else: From cfa87b1e7f3d6325c4f0ef9f4d2985948e4d99d3 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 5 Aug 2016 06:36:14 -0500 Subject: [PATCH 280/365] Add test for precedence rules This should cover both the OSC and the ansible incoming use cases. Change-Id: I3fdc83837692d31c5579d91892a387a5d1023785 --- os_client_config/tests/test_config.py | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index a0978f059..500887812 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -373,6 +373,59 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(cc.region_name, 'region2') self.assertEqual(cc.snack_type, 'cookie') + def test_get_one_cloud_precedence(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + kwargs = { + 'auth': { + 'username': 'testuser', + 'password': 'authpass', + 'project-id': 'testproject', + 'auth_url': 'http://example.com/v2', + }, + 'region_name': 'kwarg_region', + 'password': 'ansible_password', + 'arbitrary': 'value', + } + + args = dict( + auth_url='http://example.com/v2', + username='user', + password='argpass', + project_name='project', + region_name='region2', + snack_type='cookie', + ) + + options = argparse.Namespace(**args) + cc = c.get_one_cloud( + argparse=options, **kwargs) + self.assertEqual(cc.region_name, 'region2') + self.assertEqual(cc.auth['password'], 'argpass') + self.assertEqual(cc.snack_type, 'cookie') + + def test_get_one_cloud_precedence_no_argparse(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + kwargs = { + 'auth': { + 'username': 'testuser', + 'password': 'authpass', + 'project-id': 'testproject', + 'auth_url': 'http://example.com/v2', + }, + 'region_name': 'kwarg_region', + 'password': 'ansible_password', + 'arbitrary': 'value', + } + + cc = c.get_one_cloud(**kwargs) + self.assertEqual(cc.region_name, 'kwarg_region') + self.assertEqual(cc.auth['password'], 'authpass') + self.assertIsNone(cc.password) + def test_get_one_cloud_just_argparse(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) From 600a638e74d89af55fceaf4017f70269ae6e4f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Gagne=CC=81?= Date: Tue, 2 Aug 2016 15:10:43 -0400 Subject: [PATCH 281/365] Update Internap information * Add sin01 and sjc01 regions * Add support for Glance v2 Change-Id: Iaf4ad7c807a28c24040b928f65f4aadc1a234d6e --- doc/source/vendor-support.rst | 3 ++- os_client_config/vendors/internap.json | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 668ddccaa..c27d124f0 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -186,9 +186,10 @@ Region Name Human Name ams01 Amsterdam, NL da01 Dallas, TX nyj01 New York, NY +sin01 Singapore +sjc01 San Jose, CA ============== ================ -* Image API Version is 1 * Floating IPs are not supported osic diff --git a/os_client_config/vendors/internap.json b/os_client_config/vendors/internap.json index d5ad49f6d..b67fc06d4 100644 --- a/os_client_config/vendors/internap.json +++ b/os_client_config/vendors/internap.json @@ -7,10 +7,11 @@ "regions": [ "ams01", "da01", - "nyj01" + "nyj01", + "sin01", + "sjc01" ], "identity_api_version": "3", - "image_api_version": "1", "floating_ip_source": "None" } } From a6840f69ff5644065816309776365adccf017772 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 11 Aug 2016 08:14:39 -0500 Subject: [PATCH 282/365] Pop domain-id from the config if we infer values If the user specifies a project_{name,id}, then we currently infer that a domain_{name,id} is meant to be shorthand for user_domain_{name,id} and project_domain_{name,id}. However, in other contexts, domain_id is a perfectly valid value to pass to the auth plugins - such as when doing domain-scoped activities. The problem that was uncovered by the correction of argument precedence is that we didn't pop the domain-id out of the root config dict, so if a user set OS_DOMAIN_ID in the environment, we were happily setting the user and project versions, but then leaving domain_id set as well. This then meant that when we do the pass to pull valid arguments from the root dict to the auth dict, we also pulled in domain_id - which led to the error: AuthorizationFailure: Authentication cannot be scoped to multiple targets. Pick one of: project, domain, trust or unscoped Popping the value from the root dict causes the things to work as documented. Change-Id: I6d208e5ec4115d2e72d30b2fedc90a81ea754d5a --- os_client_config/config.py | 1 + os_client_config/tests/test_config.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 37a31f8d4..7fa670fed 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -562,6 +562,7 @@ class OpenStackConfig(object): for key in possible_values: if target_key in cloud['auth'] and key not in cloud['auth']: cloud['auth'][key] = cloud['auth'][target_key] + cloud.pop(target_key, None) cloud['auth'].pop(target_key, None) return cloud diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 500887812..baa35cd49 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -102,6 +102,7 @@ class TestConfig(base.TestCase): self.assertEqual('123456789', cc.auth['project_domain_id']) self.assertNotIn('domain_id', cc.auth) self.assertNotIn('domain-id', cc.auth) + self.assertNotIn('domain_id', cc) def test_get_one_cloud_domain_scoped(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], @@ -118,6 +119,7 @@ class TestConfig(base.TestCase): self.assertEqual('awesome-domain', cc.auth['user_domain_id']) self.assertEqual('awesome-domain', cc.auth['project_domain_id']) self.assertNotIn('domain_id', cc.auth) + self.assertNotIn('domain_id', cc) def test_get_one_cloud_with_hyphenated_project_id(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], From 9d757b3a9a89bdd63f56ce171b7c878ded9a4cd8 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 18 Aug 2016 11:35:50 -0500 Subject: [PATCH 283/365] Add support for configuring split-stack networks Some clouds, like OSIC and v1 of DreamCompute, have a split stack network. This means that a single Neutron Network has both an IPv4 and an IPv6 subnet, but that the IPv4 subnet is a private/RFC-1918 and the IPv6 subnet is a Global network. As any inferrance information is attached to the Network, it's impossible to properly categorize IP addresses that are on the Server in such a scenario. Add support for ipv4 and ipv6 versions of the current routes_externally config value, with each of them defaulting to the value of the un-specialized routes_externally. Change-Id: I1e87a1423d20eac31175f44f5f7b38dfcf3a11cb --- doc/source/network-config.rst | 13 ++++++++++++- os_client_config/cloud_config.py | 24 ++++++++++++++++++++++++ os_client_config/config.py | 8 ++++++++ os_client_config/tests/base.py | 8 ++++++++ os_client_config/tests/test_config.py | 16 +++++++++++++--- 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/doc/source/network-config.rst b/doc/source/network-config.rst index 5a27b8e0e..09571804e 100644 --- a/doc/source/network-config.rst +++ b/doc/source/network-config.rst @@ -19,9 +19,15 @@ allows configuring a list of network metadata. default_interface: true - name: green routes_externally: false - - name: purple + - name: yellow routes_externally: false nat_destination: true + - name: chartreuse + routes_externally: false + routes_ipv6_externally: true + - name: aubergine + routes_ipv4_externally: false + routes_ipv6_externally: true Every entry must have a name field, which can hold either the name or the id of the network. @@ -33,6 +39,11 @@ be an RFC1918 address. In either case, it's provides IPs to servers that things not on the cloud can use. This value defaults to `false`, which indicates only servers on the same network can talk to it. +`routes_ipv4_externally` and `routes_ipv6_externally` are boolean fields to +help handle `routes_externally` in the case where a network has a split stack +with different values for IPv4 and IPv6. Either entry, if not given, defaults +to the value of `routes_externally`. + `default_interface` is a boolean field that indicates that the network is the one that programs should use. It defaults to false. An example of needing to use this value is a cloud with two private networks, and where a user is diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 08c739a4d..1846f5460 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -452,12 +452,36 @@ class CloudConfig(object): net['name'] for net in self.config['networks'] if net['routes_externally']] + def get_external_ipv4_networks(self): + """Get list of network names for external IPv4 networks.""" + return [ + net['name'] for net in self.config['networks'] + if net['routes_ipv4_externally']] + + def get_external_ipv6_networks(self): + """Get list of network names for external IPv6 networks.""" + return [ + net['name'] for net in self.config['networks'] + if net['routes_ipv6_externally']] + def get_internal_networks(self): """Get list of network names for internal networks.""" return [ net['name'] for net in self.config['networks'] if not net['routes_externally']] + def get_internal_ipv4_networks(self): + """Get list of network names for internal IPv4 networks.""" + return [ + net['name'] for net in self.config['networks'] + if not net['routes_ipv4_externally']] + + def get_internal_ipv6_networks(self): + """Get list of network names for internal IPv6 networks.""" + return [ + net['name'] for net in self.config['networks'] + if not net['routes_ipv6_externally']] + def get_default_network(self): """Get network used for default interactions.""" for net in self.config['networks']: diff --git a/os_client_config/config.py b/os_client_config/config.py index 7fa670fed..a51893202 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -520,6 +520,14 @@ class OpenStackConfig(object): nat_destination=get_boolean(net.get('nat_destination')), default_interface=get_boolean(net.get('default_interface')), ) + # routes_ipv4_externally defaults to the value of routes_externally + network['routes_ipv4_externally'] = get_boolean( + net.get( + 'routes_ipv4_externally', network['routes_externally'])) + # routes_ipv6_externally defaults to the value of routes_externally + network['routes_ipv6_externally'] = get_boolean( + net.get( + 'routes_ipv6_externally', network['routes_externally'])) networks.append(network) for key in ('external_network', 'internal_network'): diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index d046a941f..85a42c593 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -113,6 +113,14 @@ USER_CONF = { 'name': 'another-private', 'routes_externally': False, 'nat_destination': True, + }, { + 'name': 'split-default', + 'routes_externally': True, + 'routes_ipv4_externally': False, + }, { + 'name': 'split-no-default', + 'routes_ipv6_externally': False, + 'routes_ipv4_externally': True, }], 'region_name': 'test-region', }, diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index baa35cd49..b9dcb212a 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -194,11 +194,19 @@ class TestConfig(base.TestCase): vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud('_test-cloud-networks_') self.assertEqual( - ['a-public', 'another-public'], cc.get_external_networks()) + ['a-public', 'another-public', 'split-default'], + cc.get_external_networks()) self.assertEqual( - ['a-private', 'another-private'], cc.get_internal_networks()) + ['a-private', 'another-private', 'split-no-default'], + cc.get_internal_networks()) self.assertEqual('another-private', cc.get_nat_destination()) self.assertEqual('another-public', cc.get_default_network()) + self.assertEqual( + ['a-public', 'another-public', 'split-no-default'], + cc.get_external_ipv4_networks()) + self.assertEqual( + ['a-public', 'another-public', 'split-default'], + cc.get_external_ipv6_networks()) def test_get_one_cloud_no_networks(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], @@ -875,7 +883,9 @@ class TestBackwardsCompatibility(base.TestCase): expected = { 'networks': [ {'name': 'private', 'routes_externally': False, - 'nat_destination': False, 'default_interface': False}, + 'nat_destination': False, 'default_interface': False, + 'routes_ipv4_externally': False, + 'routes_ipv6_externally': False}, ] } self.assertEqual(expected, result) From 17f6847c20bc4b8ea1c8fc48f81bf1eca4ddb0f5 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Sat, 20 Aug 2016 16:14:36 -0500 Subject: [PATCH 284/365] Precedence final solution * Revert most of 'fixed_argparse change' from 1.19.1 * Create a new _validate_auth_correctly() method that contains the logic from 1.19.0 * Create a new get_one_cloud_osc() method for use by OSC to get the correct argument precedence without disrupting anyone else Change-Id: Iae86cc4e267f23dbe8d010688a288db5514f329d --- os_client_config/config.py | 220 +++++++++++++++++++------- os_client_config/tests/test_config.py | 36 +++++ 2 files changed, 200 insertions(+), 56 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 37a31f8d4..c799de0b0 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -836,7 +836,7 @@ class OpenStackConfig(object): config['auth']['token'] = 'notused' return loading.get_plugin_loader(config['auth_type']) - def _validate_auth_ksc(self, config, cloud, fixed_argparse): + def _validate_auth_ksc(self, config, cloud): try: import keystoneclient.auth as ksc_auth except ImportError: @@ -847,22 +847,19 @@ class OpenStackConfig(object): config['auth_type']).get_options() for p_opt in plugin_options: - # if it's in argparse, it was passed on the command line and wins # if it's in config.auth, win, kill it from config dict # if it's in config and not in config.auth, move it # deprecated loses to current # provided beats default, deprecated or not winning_value = self._find_winning_auth_value( - p_opt, fixed_argparse) - if winning_value: - found_in_argparse = True - else: - found_in_argparse = False + p_opt, + config['auth'], + ) + if not winning_value: winning_value = self._find_winning_auth_value( - p_opt, config['auth']) - if not winning_value: - winning_value = self._find_winning_auth_value( - p_opt, config) + p_opt, + config, + ) # if the plugin tells us that this value is required # then error if it's doesn't exist now @@ -878,12 +875,7 @@ class OpenStackConfig(object): # Clean up after ourselves for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: opt = opt.replace('-', '_') - # don't do this if the value came from argparse, because we - # don't (yet) know if the value in not-auth came from argparse - # overlay or from someone passing in a dict to kwargs - # TODO(mordred) Fix that data path too - if not found_in_argparse: - config.pop(opt, None) + config.pop(opt, None) config['auth'].pop(opt, None) if winning_value: @@ -897,49 +889,78 @@ class OpenStackConfig(object): return config - def _validate_auth(self, config, loader, fixed_argparse): + def _validate_auth(self, config, loader): # May throw a keystoneauth1.exceptions.NoMatchingPlugin plugin_options = loader.get_options() for p_opt in plugin_options: - # if it's in argparse, it was passed on the command line and wins # if it's in config.auth, win, kill it from config dict # if it's in config and not in config.auth, move it # deprecated loses to current # provided beats default, deprecated or not winning_value = self._find_winning_auth_value( - p_opt, fixed_argparse) - if winning_value: - found_in_argparse = True - else: - found_in_argparse = False + p_opt, + config['auth'], + ) + if not winning_value: winning_value = self._find_winning_auth_value( - p_opt, config['auth']) - if not winning_value: - winning_value = self._find_winning_auth_value( - p_opt, config) + p_opt, + config, + ) - # Clean up after ourselves - for opt in [p_opt.name] + [o.name for o in p_opt.deprecated]: - opt = opt.replace('-', '_') - # don't do this if the value came from argparse, because we - # don't (yet) know if the value in not-auth came from argparse - # overlay or from someone passing in a dict to kwargs - # TODO(mordred) Fix that data path too - if not found_in_argparse: - config.pop(opt, None) - config['auth'].pop(opt, None) + config = self._clean_up_after_ourselves( + config, + p_opt, + winning_value, + ) - if winning_value: - # Prefer the plugin configuration dest value if the value's key - # is marked as depreciated. - if p_opt.dest is None: - config['auth'][p_opt.name.replace('-', '_')] = ( - winning_value) - else: - config['auth'][p_opt.dest] = winning_value + return config + def _validate_auth_correctly(self, config, loader): + # May throw a keystoneauth1.exceptions.NoMatchingPlugin + + plugin_options = loader.get_options() + + for p_opt in plugin_options: + # if it's in config, win, move it and kill it from config dict + # if it's in config.auth but not in config it's good + # deprecated loses to current + # provided beats default, deprecated or not + winning_value = self._find_winning_auth_value( + p_opt, + config, + ) + if not winning_value: + winning_value = self._find_winning_auth_value( + p_opt, + config['auth'], + ) + + config = self._clean_up_after_ourselves( + config, + p_opt, + winning_value, + ) + + return config + + def _clean_up_after_ourselves(self, config, p_opt, winning_value): + + # Clean up after ourselves + for opt in [p_opt.name] + [o.name for o in p_opt.deprecated]: + opt = opt.replace('-', '_') + config.pop(opt, None) + config['auth'].pop(opt, None) + + if winning_value: + # Prefer the plugin configuration dest value if the value's key + # is marked as depreciated. + if p_opt.dest is None: + config['auth'][p_opt.name.replace('-', '_')] = ( + winning_value) + else: + config['auth'][p_opt.dest] = winning_value return config def magic_fixes(self, config): @@ -998,11 +1019,6 @@ class OpenStackConfig(object): """ args = self._fix_args(kwargs, argparse=argparse) - # Run the fix just for argparse by itself. We need to - # have a copy of the argparse options separately from - # any merged copied later in validate_auth so that we - # can determine precedence - fixed_argparse = self._fix_args(argparse=argparse) if cloud is None: if 'cloud' in args: @@ -1042,7 +1058,7 @@ class OpenStackConfig(object): if validate: try: loader = self._get_auth_loader(config) - config = self._validate_auth(config, loader, fixed_argparse) + config = self._validate_auth(config, loader) auth_plugin = loader.load_from_options(**config['auth']) except Exception as e: # We WANT the ksa exception normally @@ -1052,8 +1068,7 @@ class OpenStackConfig(object): self.log.debug("Deferring keystone exception: {e}".format(e=e)) auth_plugin = None try: - config = self._validate_auth_ksc( - config, cloud, fixed_argparse) + config = self._validate_auth_ksc(config, cloud) except Exception: raise e else: @@ -1074,12 +1089,105 @@ class OpenStackConfig(object): else: cloud_name = str(cloud) return cloud_config.CloudConfig( - name=cloud_name, region=config['region_name'], + name=cloud_name, + region=config['region_name'], config=self._normalize_keys(config), force_ipv4=force_ipv4, auth_plugin=auth_plugin, openstack_config=self - ) + ) + + def get_one_cloud_osc( + self, + cloud=None, + validate=True, + argparse=None, + **kwargs + ): + """Retrieve a single cloud configuration and merge additional options + + :param string cloud: + The name of the configuration to load from clouds.yaml + :param boolean validate: + Validate the config. Setting this to False causes no auth plugin + to be created. It's really only useful for testing. + :param Namespace argparse: + An argparse Namespace object; allows direct passing in of + argparse options to be added to the cloud config. Values + of None and '' will be removed. + :param region_name: Name of the region of the cloud. + :param kwargs: Additional configuration options + + :raises: keystoneauth1.exceptions.MissingRequiredOptions + on missing required auth parameters + """ + + args = self._fix_args(kwargs, argparse=argparse) + + if cloud is None: + if 'cloud' in args: + cloud = args['cloud'] + else: + cloud = self.default_cloud + + config = self._get_base_cloud_config(cloud) + + # Get region specific settings + if 'region_name' not in args: + args['region_name'] = '' + region = self._get_region(cloud=cloud, region_name=args['region_name']) + args['region_name'] = region['name'] + region_args = copy.deepcopy(region['values']) + + # Regions is a list that we can use to create a list of cloud/region + # objects. It does not belong in the single-cloud dict + config.pop('regions', None) + + # Can't just do update, because None values take over + for arg_list in region_args, args: + for (key, val) in iter(arg_list.items()): + if val is not None: + if key == 'auth' and config[key] is not None: + config[key] = _auth_update(config[key], val) + else: + config[key] = val + + config = self.magic_fixes(config) + + # NOTE(dtroyer): OSC needs a hook into the auth args before the + # plugin is loaded in order to maintain backward- + # compatible behaviour + config = self.auth_config_hook(config) + + if validate: + loader = self._get_auth_loader(config) + config = self._validate_auth_correctly(config, loader) + auth_plugin = loader.load_from_options(**config['auth']) + else: + auth_plugin = None + + # If any of the defaults reference other values, we need to expand + for (key, value) in config.items(): + if hasattr(value, 'format'): + config[key] = value.format(**config) + + force_ipv4 = config.pop('force_ipv4', self.force_ipv4) + prefer_ipv6 = config.pop('prefer_ipv6', True) + if not prefer_ipv6: + force_ipv4 = True + + if cloud is None: + cloud_name = '' + else: + cloud_name = str(cloud) + return cloud_config.CloudConfig( + name=cloud_name, + region=config['region_name'], + config=self._normalize_keys(config), + force_ipv4=force_ipv4, + auth_plugin=auth_plugin, + openstack_config=self, + ) @staticmethod def set_one_cloud(config_file, cloud, set_config=None): diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 500887812..e1113886b 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -402,6 +402,42 @@ class TestConfigArgparse(base.TestCase): cc = c.get_one_cloud( argparse=options, **kwargs) self.assertEqual(cc.region_name, 'region2') + self.assertEqual(cc.auth['password'], 'authpass') + self.assertEqual(cc.snack_type, 'cookie') + + def test_get_one_cloud_precedence_osc(self): + c = config.OpenStackConfig( + config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + ) + + kwargs = { + 'auth': { + 'username': 'testuser', + 'password': 'authpass', + 'project-id': 'testproject', + 'auth_url': 'http://example.com/v2', + }, + 'region_name': 'kwarg_region', + 'password': 'ansible_password', + 'arbitrary': 'value', + } + + args = dict( + auth_url='http://example.com/v2', + username='user', + password='argpass', + project_name='project', + region_name='region2', + snack_type='cookie', + ) + + options = argparse.Namespace(**args) + cc = c.get_one_cloud_osc( + argparse=options, + **kwargs + ) + self.assertEqual(cc.region_name, 'region2') self.assertEqual(cc.auth['password'], 'argpass') self.assertEqual(cc.snack_type, 'cookie') From 72c1cd9c41b9e0862d3bde1bea84a60e41dfefd0 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 25 Aug 2016 14:32:46 -0500 Subject: [PATCH 285/365] Clean up vendor support list The IBM Cloud isn't really a thing at that address yet (jumped the gun) OSIC probably shouldn't have been added either (turns out there are like 8 OSIC clouds currently, and they're test clouds. Also, update the location on the RegionOne clouds, and rename "Human Name" to "Location" - which is the useful information in that column anyway. Change-Id: I04451836330aacc3e2b91cfbe7d7d9bba7a47346 --- doc/source/vendor-support.rst | 84 ++++++++++++----------------------- 1 file changed, 29 insertions(+), 55 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index c27d124f0..cfdc02eeb 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -31,7 +31,7 @@ auro https://api.auro.io:5000/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ van1 Vancouver, BC ============== ================ @@ -44,7 +44,7 @@ catalyst https://api.cloud.catalyst.net.nz:5000/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ nz-por-1 Porirua, NZ nz_wlg_2 Wellington, NZ @@ -60,7 +60,7 @@ citycloud https://identity1.citycloud.com:5000/v3/ ============== ================ -Region Name Human Name +Region Name Location ============== ================ Buf1 Buffalo, NY Fra1 Frankfurt, DE @@ -80,7 +80,7 @@ conoha https://identity.%(region_name)s.conoha.io ============== ================ -Region Name Human Name +Region Name Location ============== ================ tyo1 Tokyo, JP sin1 Singapore @@ -95,7 +95,7 @@ datacentred https://compute.datacentred.io:5000 ============== ================ -Region Name Human Name +Region Name Location ============== ================ sal01 Manchester, UK ============== ================ @@ -108,9 +108,9 @@ dreamcompute https://iad2.dream.io:5000 ============== ================ -Region Name Human Name +Region Name Location ============== ================ -RegionOne Region One +RegionOne Ashburn, VA ============== ================ * Identity API Version is 3 @@ -125,9 +125,9 @@ Deprecated, please use dreamcompute https://keystone.dream.io/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ -RegionOne Region One +RegionOne Ashburn, VA ============== ================ * Images must be in `raw` format @@ -140,9 +140,9 @@ elastx https://ops.elastx.net:5000/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ -regionOne Region One +regionOne Stockholm, SE ============== ================ * Public IPv4 is provided via NAT with Neutron Floating IP @@ -153,7 +153,7 @@ entercloudsuite https://api.entercloudsuite.com/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ nl-ams1 Amsterdam, NL it-mil1 Milan, IT @@ -162,26 +162,13 @@ de-fra1 Frankfurt, DE * Volume API Version is 1 -ibmcloud --------- - -https://identity.open.softlayer.com - -============== ================ -Region Name Human Name -============== ================ -london London, UK -============== ================ - -* Public IPv4 is provided via NAT with Neutron Floating IP - internap -------- https://identity.api.cloud.iweb.com/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ ams01 Amsterdam, NL da01 Dallas, TX @@ -192,26 +179,13 @@ sjc01 San Jose, CA * Floating IPs are not supported -osic ----- - -https://cloud1.osic.org:5000 - -============== ================= -Region Name Human Name -============== ================= -RegionOne RegionOne -============== ================= - -* Public IPv4 is provided via NAT with Neutron Floating IP - ovh --- https://auth.cloud.ovh.net/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ BHS1 Beauharnois, QC SBG1 Strassbourg, FR @@ -227,14 +201,14 @@ rackspace https://identity.api.rackspacecloud.com/v2.0/ ============== ================ -Region Name Human Name +Region Name Location ============== ================ -DFW Dallas +DFW Dallas, TX HKG Hong Kong IAD Washington, D.C. -LON London -ORD Chicago -SYD Sydney +LON London, UK +ORD Chicago, IL +SYD Sydney, NSW ============== ================ * Database Service Type is `rax:database` @@ -256,7 +230,7 @@ switchengines https://keystone.cloud.switch.ch:5000/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ LS Lausanne, CH ZH Zurich, CH @@ -272,9 +246,9 @@ ultimum https://console.ultimum-cloud.com:5000/v2.0 ============== ================ -Region Name Human Name +Region Name Location ============== ================ -RegionOne Region One +RegionOne Prague, CZ ============== ================ * Volume API Version is 1 @@ -285,10 +259,10 @@ unitedstack https://identity.api.ustack.com/v3 ============== ================ -Region Name Human Name +Region Name Location ============== ================ -bj1 Beijing -gd1 Guangdong +bj1 Beijing, CN +gd1 Guangdong, CN ============== ================ * Identity API Version is 3 @@ -301,9 +275,9 @@ vexxhost http://auth.vexxhost.net ============== ================ -Region Name Human Name +Region Name Location ============== ================ -ca-ymq-1 Montreal +ca-ymq-1 Montreal, QC ============== ================ * DNS API Version is 1 @@ -315,9 +289,9 @@ zetta https://identity.api.zetta.io/v3 ============== ================ -Region Name Human Name +Region Name Location ============== ================ -no-osl1 Oslo +no-osl1 Oslo, NO ============== ================ * DNS API Version is 2 From 2b52bcf6943369fe96db3aefe2ee7473dd51b934 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Mon, 29 Aug 2016 13:07:17 -0500 Subject: [PATCH 286/365] Add prompting for KSA options Teach OpenStackConfig to prompt the user for KSA plugin options that have no value but have a prompt string defined. * Add pw_func argument to __init__() to be used as the callback for prompting the user. The default is None which skips the prompt step. * Add option_prompt() method to perform the checks for prompting, call the callback and save the returned value. This is public to handle cases where simply passing in a callback is insufficient for the prompt mechanism. Related-Bug: #1617384 Change-Id: I5faa86e94d6f71282ac270e2acfbd3016638c780 --- os_client_config/config.py | 23 ++++++++++++++++- os_client_config/tests/test_config.py | 37 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 586f0a79c..54048aad1 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -173,7 +173,8 @@ class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, override_defaults=None, force_ipv4=None, - envvar_prefix=None, secure_files=None): + envvar_prefix=None, secure_files=None, + pw_func=None): self.log = _log.setup_logging(__name__) self._config_files = config_files or CONFIG_FILES @@ -288,6 +289,10 @@ class OpenStackConfig(object): # Flag location to hold the peeked value of an argparse timeout value self._argv_timeout = False + # Save the password callback + # password = self._pw_callback(prompt="Password: ") + self._pw_callback = pw_func + def get_extra_config(self, key, defaults=None): """Fetch an arbitrary extra chunk of config, laying in defaults. @@ -924,6 +929,9 @@ class OpenStackConfig(object): winning_value, ) + # See if this needs a prompting + config = self.option_prompt(config, p_opt) + return config def _validate_auth_correctly(self, config, loader): @@ -952,6 +960,19 @@ class OpenStackConfig(object): winning_value, ) + # See if this needs a prompting + config = self.option_prompt(config, p_opt) + + return config + + def option_prompt(self, config, p_opt): + """Prompt user for option that requires a value""" + if ( + p_opt.prompt is not None and + p_opt.dest not in config['auth'] and + self._pw_callback is not None + ): + config['auth'][p_opt.dest] = self._pw_callback(p_opt.prompt) return config def _clean_up_after_ourselves(self, config, p_opt, winning_value): diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 5bcf7660e..5cded544a 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -27,6 +27,11 @@ from os_client_config import exceptions from os_client_config.tests import base +def prompt_for_password(prompt=None): + """Fake prompt function that just returns a constant string""" + return 'promptpass' + + class TestConfig(base.TestCase): def test_get_all_clouds(self): @@ -787,6 +792,38 @@ class TestConfigArgparse(base.TestCase): self.assertNotIn('http_timeout', cloud.config) +class TestConfigPrompt(base.TestCase): + + def setUp(self): + super(TestConfigPrompt, self).setUp() + + self.args = dict( + auth_url='http://example.com/v2', + username='user', + project_name='project', + # region_name='region2', + auth_type='password', + ) + + self.options = argparse.Namespace(**self.args) + + def test_get_one_cloud_prompt(self): + c = config.OpenStackConfig( + config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml], + pw_func=prompt_for_password, + ) + + # This needs a cloud definition without a password. + # If this starts failing unexpectedly check that the cloud_yaml + # and/or vendor_yaml do not have a password in the selected cloud. + cc = c.get_one_cloud( + cloud='_test_cloud_no_vendor', + argparse=self.options, + ) + self.assertEqual('promptpass', cc.auth['password']) + + class TestConfigDefault(base.TestCase): def setUp(self): From 42885dc94176eecca1619d7e9c33b5ab3d9a2c0a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 29 Aug 2016 17:49:46 -0500 Subject: [PATCH 287/365] Go ahead and handle YAML list in region_name The right thing is to use "regions" as a yaml list. However, it's even more right to be friendly to people - we emit a warning about region_name as a list anyway. Change-Id: Ia0b27ef8d1d52c655c2736a97bd5e59a4a2fe9d8 --- os_client_config/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 54048aad1..6eed0f0cd 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -388,7 +388,10 @@ class OpenStackConfig(object): if 'regions' in config: return self._expand_regions(config['regions']) elif 'region_name' in config: - regions = config['region_name'].split(',') + if isinstance(config['region_name'], list): + regions = config['region_name'] + else: + regions = config['region_name'].split(',') if len(regions) > 1: warnings.warn( "Comma separated lists in region_name are deprecated." From 8b7859e21e64027d20f158737bbf70bbe409b847 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 1 Sep 2016 09:10:03 -0500 Subject: [PATCH 288/365] Split auth plugin loading into its own method In case someone wants to do validate=False at get_one_cloud time, but still would like to get an auth plugin, having this block be its own method is handy. Change-Id: I26137391e67d60d8737473d3d4c6ed15dad53af1 --- os_client_config/config.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 6eed0f0cd..7a70daaf6 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -1031,6 +1031,24 @@ class OpenStackConfig(object): return config + def load_auth_plugin(self, config, cloud): + try: + loader = self._get_auth_loader(config) + config = self._validate_auth(config, loader) + auth_plugin = loader.load_from_options(**config['auth']) + except Exception as e: + # We WANT the ksa exception normally + # but OSC can't handle it right now, so we try deferring + # to ksc. If that ALSO fails, it means there is likely + # a deeper issue, so we assume the ksa error was correct + self.log.debug("Deferring keystone exception: {e}".format(e=e)) + auth_plugin = None + try: + config = self._validate_auth_ksc(config, cloud) + except Exception: + raise e + return auth_plugin + def get_one_cloud(self, cloud=None, validate=True, argparse=None, **kwargs): """Retrieve a single cloud configuration and merge additional options @@ -1089,21 +1107,7 @@ class OpenStackConfig(object): config = self.auth_config_hook(config) if validate: - try: - loader = self._get_auth_loader(config) - config = self._validate_auth(config, loader) - auth_plugin = loader.load_from_options(**config['auth']) - except Exception as e: - # We WANT the ksa exception normally - # but OSC can't handle it right now, so we try deferring - # to ksc. If that ALSO fails, it means there is likely - # a deeper issue, so we assume the ksa error was correct - self.log.debug("Deferring keystone exception: {e}".format(e=e)) - auth_plugin = None - try: - config = self._validate_auth_ksc(config, cloud) - except Exception: - raise e + auth_plugin = self.load_auth_plugin(config, cloud) else: auth_plugin = None From 86b100e400ad9dbda018d4be7f97ce4582737fe7 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 1 Sep 2016 18:08:15 -0500 Subject: [PATCH 289/365] Add ability to configure Session constructor Sometimes it's useful to wrapp the keystoneauth.Session object. OSC has a KeystoneSession object that injects timing data. shade is considering one that wraps external calls in shade's TaskManager. Allow for passing in a callable that will return a Session. Almost no people will want to use this - it's a super advanced kind of thing. Change-Id: Ib64260916695e9fbea437862cd669a4fb85ec9e4 --- os_client_config/cloud_config.py | 5 +++-- os_client_config/config.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 1846f5460..ae5da3a5a 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -70,7 +70,7 @@ def _make_key(key, service_type): class CloudConfig(object): def __init__(self, name, region, config, force_ipv4=False, auth_plugin=None, - openstack_config=None): + openstack_config=None, session_constructor=None): self.name = name self.region = region self.config = config @@ -79,6 +79,7 @@ class CloudConfig(object): self._auth = auth_plugin self._openstack_config = openstack_config self._keystone_session = None + self._session_constructor = session_constructor or session.Session def __getattr__(self, key): """Return arbitrary attributes.""" @@ -196,7 +197,7 @@ class CloudConfig(object): " since verify=False".format( cloud=self.name, region=self.region)) requestsexceptions.squelch_warnings(insecure_requests=not verify) - self._keystone_session = session.Session( + self._keystone_session = self._session_constructor( auth=self._auth, verify=verify, cert=cert, diff --git a/os_client_config/config.py b/os_client_config/config.py index 6eed0f0cd..0c11230ef 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -174,8 +174,9 @@ class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, override_defaults=None, force_ipv4=None, envvar_prefix=None, secure_files=None, - pw_func=None): + pw_func=None, session_constructor=None): self.log = _log.setup_logging(__name__) + self._session_constructor = session_constructor self._config_files = config_files or CONFIG_FILES self._secure_files = secure_files or SECURE_FILES @@ -1127,7 +1128,8 @@ class OpenStackConfig(object): config=self._normalize_keys(config), force_ipv4=force_ipv4, auth_plugin=auth_plugin, - openstack_config=self + openstack_config=self, + session_constructor=self._session_constructor, ) def get_one_cloud_osc( From 91eb5e062a5ea70a65edb08dd22d053b26a3d0f4 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 2 Sep 2016 09:38:36 -0400 Subject: [PATCH 290/365] Update reno for stable/newton Change-Id: I829a65b2104ec3c039859dce2594b701981b1fa3 --- releasenotes/source/index.rst | 1 + releasenotes/source/newton.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/newton.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 2f4234a88..f708ec8d3 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -9,6 +9,7 @@ Contents :maxdepth: 2 unreleased + newton mitaka Indices and tables diff --git a/releasenotes/source/newton.rst b/releasenotes/source/newton.rst new file mode 100644 index 000000000..97036ed25 --- /dev/null +++ b/releasenotes/source/newton.rst @@ -0,0 +1,6 @@ +=================================== + Newton Series Release Notes +=================================== + +.. release-notes:: + :branch: origin/stable/newton From f91a75426b877edc1f0f2e0abe4db9f40e837242 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 2 Sep 2016 11:21:24 -0500 Subject: [PATCH 291/365] Don't build releasenotes in normal docs build The release notes system exists in parallel and publishes to http://docs.openstack.org/releasenotes/os-client-config/. When we build them in the normal doc build, it causes problems for distro packagers. Change-Id: I6b084a1ad6836beac991d03c5f134203512150ac --- doc/source/releasenotes.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/releasenotes.rst b/doc/source/releasenotes.rst index 2a4bceb4e..9f41b7e14 100644 --- a/doc/source/releasenotes.rst +++ b/doc/source/releasenotes.rst @@ -2,4 +2,5 @@ Release Notes ============= -.. release-notes:: +Release notes for `os-client-config` can be found at +http://docs.openstack.org/releasenotes/os-client-config/ From c6d2aeada4d9074f185f9748a81f7c651614e347 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 15 Sep 2016 16:41:17 -0500 Subject: [PATCH 292/365] Don't create envvars cloud if cloud or region are set OS_CLOUD and OS_REGION_NAME are both selectors. If they are the only values, we should not be creating an envvars cloud. Change-Id: I1b7c8d7e3b6c300bd2c85ab482a22411370e4d5f --- os_client_config/config.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 646e55ddf..aa6ef7ebf 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -105,12 +105,11 @@ def _get_os_environ(envvar_prefix=None): for k in environkeys: newkey = k.split('_', 1)[-1].lower() ret[newkey] = os.environ[k] - # If the only environ key is region name, don't make a cloud, because - # it's being used as a cloud selector - if not environkeys or ( - len(environkeys) == 1 and 'region_name' in ret): - return None - return ret + # If the only environ keys are cloud and region_name, don't return anything + # because they are cloud selectors + if set(environkeys) - set(['OS_CLOUD', 'OS_REGION_NAME']): + return ret + return None def _merge_clouds(old_dict, new_dict): From eec981c96e3c1824a547c8baa6fc5b08dba57624 Mon Sep 17 00:00:00 2001 From: avnish Date: Tue, 20 Sep 2016 11:39:00 +0530 Subject: [PATCH 293/365] modify the home-page info with the developer documentation Change-Id: I72a16e15e61c8e6511b96114cc74e3feb7dc6fd3 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 07ac625ac..f8dcbb096 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://www.openstack.org/ +home-page = http://docs.openstack.org/developer/os-client-config/ classifier = Environment :: OpenStack Intended Audience :: Information Technology From 0a3e056cce03cdb3dbd62d8eddfaa448d5a0ad5f Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Mon, 26 Sep 2016 17:22:27 +0200 Subject: [PATCH 294/365] Fix AttributeError in `get_config` Change-Id: I52bdc44800da6c1393a69c4faf96375235ef98bb Closes-Bug: #1627690 --- os_client_config/__init__.py | 2 +- os_client_config/config.py | 5 ++++- os_client_config/tests/test_init.py | 33 +++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 os_client_config/tests/test_init.py diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index 09d74423b..e8d7fc0f8 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -26,7 +26,7 @@ __version__ = pbr.version.VersionInfo('os_client_config').version_string() def get_config(service_key=None, options=None, **kwargs): config = OpenStackConfig() if options: - config.register_argparse_options(options, sys.argv, service_key) + config.register_argparse_arguments(options, sys.argv, service_key) parsed_options = options.parse_known_args(sys.argv) else: parsed_options = None diff --git a/os_client_config/config.py b/os_client_config/config.py index 646e55ddf..01f659e49 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -641,7 +641,7 @@ class OpenStackConfig(object): # completely broken return cloud - def register_argparse_arguments(self, parser, argv, service_keys=[]): + def register_argparse_arguments(self, parser, argv, service_keys=None): """Register all of the common argparse options needed. Given an argparse parser, register the keystoneauth Session arguments, @@ -660,6 +660,9 @@ class OpenStackConfig(object): is requested """ + if service_keys is None: + service_keys = [] + # Fix argv in place - mapping any keys with embedded _ in them to - _fix_argv(argv) diff --git a/os_client_config/tests/test_init.py b/os_client_config/tests/test_init.py new file mode 100644 index 000000000..15d57f717 --- /dev/null +++ b/os_client_config/tests/test_init.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse + +import os_client_config +from os_client_config.tests import base + + +class TestInit(base.TestCase): + def test_get_config_without_arg_parser(self): + cloud_config = os_client_config.get_config(options=None) + self.assertIsInstance( + cloud_config, + os_client_config.cloud_config.CloudConfig + ) + + def test_get_config_with_arg_parser(self): + cloud_config = os_client_config.get_config( + options=argparse.ArgumentParser()) + self.assertIsInstance( + cloud_config, + os_client_config.cloud_config.CloudConfig + ) From bd434bc6268d7d4b872cbf17cf23a4fd5dc1a855 Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Mon, 26 Sep 2016 17:39:56 +0200 Subject: [PATCH 295/365] List py35 in the default tox env list We really should run py35 tests when we run "tox" without any arguments. I2a4a6ca01d7cca83f594008960c878a18ca08e8e is going to make the py35 job voting. Change-Id: Ibd77e39c53f00357344be8acc2949e1bc1adcc84 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d2aac05bd..cedf47857 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py34,py27,pypy,pep8 +envlist = py34,py35,py27,pypy,pep8 skipsdist = True [testenv] From 2d78f1ae940d4b2810437ea3c73379413c3ef6ef Mon Sep 17 00:00:00 2001 From: Tony Xu Date: Mon, 26 Sep 2016 23:51:53 +0800 Subject: [PATCH 296/365] Update homepage with developer documentation page Change-Id: Id7df70e3cfaa29218c2e2badefcbc8a296d86f8d --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 07ac625ac..f8dcbb096 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://www.openstack.org/ +home-page = http://docs.openstack.org/developer/os-client-config/ classifier = Environment :: OpenStack Intended Audience :: Information Technology From afe54856b42e23e6d74da345f4870a95f646bc99 Mon Sep 17 00:00:00 2001 From: Cao Xuan Hoang Date: Wed, 28 Sep 2016 10:09:49 +0700 Subject: [PATCH 297/365] Using assertIsNone() instead of assertEqual(None, ...) Following OpenStack Style Guidelines[1]: [H203] Unit test assertions tend to give better messages for more specific assertions. As a result, assertIsNone(...) is preferred over assertEqual(None, ...) and assertIs(None, ...) [1] http://docs.openstack.org/developer/hacking/#unit-tests-and-assertraises Change-Id: I4ce1745a90b043ea342fb157683b01f862c1bc3d --- os_client_config/tests/test_config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 5cded544a..0ef8da2f2 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -753,8 +753,8 @@ class TestConfigArgparse(base.TestCase): opts, _remain = parser.parse_known_args(args) self.assertEqual(opts.os_service_type, 'network') self.assertEqual(opts.os_endpoint_type, 'admin') - self.assertEqual(opts.os_network_service_type, None) - self.assertEqual(opts.os_network_api_version, None) + self.assertIsNone(opts.os_network_service_type) + self.assertIsNone(opts.os_network_api_version) self.assertEqual(opts.network_api_version, '4') cloud = c.get_one_cloud(argparse=opts, verify=False) self.assertEqual(cloud.config['service_type'], 'network') @@ -776,12 +776,12 @@ class TestConfigArgparse(base.TestCase): parser, args, ['compute', 'network', 'volume']) opts, _remain = parser.parse_known_args(args) self.assertEqual(opts.os_network_service_type, 'badtype') - self.assertEqual(opts.os_compute_service_type, None) - self.assertEqual(opts.os_volume_service_type, None) + self.assertIsNone(opts.os_compute_service_type) + self.assertIsNone(opts.os_volume_service_type) self.assertEqual(opts.os_service_type, 'compute') self.assertEqual(opts.os_compute_service_name, 'cloudServers') self.assertEqual(opts.os_endpoint_type, 'admin') - self.assertEqual(opts.os_network_api_version, None) + self.assertIsNone(opts.os_network_api_version) self.assertEqual(opts.network_api_version, '4') cloud = c.get_one_cloud(argparse=opts, verify=False) self.assertEqual(cloud.config['service_type'], 'compute') From f484736b7d087115b8330ae794c101f7f8e7b926 Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Thu, 6 Oct 2016 16:00:46 +0200 Subject: [PATCH 298/365] cloud_config:get_session_endpoint: catch Keystone EndpointNotFound The docstring of the `get_session_endpoint` says ":returns: Endpoint for the service, or None if not found" but apparently the `None` part was forgotten. This leads to this kind of spectacular traceback where the exception bubles up (even through Shade): Traceback (most recent call last): File ".tox/run/bin/ospurge", line 11, in load_entry_point('ospurge', 'console_scripts', 'ospurge')() File "/path/ospurge/main.py", line 154, in main for resource in resource_manager.list(): File "/path/ospurge/resources/swift.py", line 15, in list for container in self.cloud.list_containers(): File "/path/pkg/shade/openstackcloud.py", line 4909, in list_containers full_listing=full_listing)) File "/path/pkg/shade/task_manager.py", line 244, in submit_task return task.wait(raw) File "/path/pkg/shade/task_manager.py", line 121, in wait super(Task, self).wait() File "/path/pkg/shade/task_manager.py", line 96, in wait self._traceback) File "/path/pkg/six.py", line 686, in reraise raise value File "/path/pkg/shade/task_manager.py", line 105, in run self.done(self.main(client)) File "/path/pkg/shade/_tasks.py", line 549, in main return client.swift_client.get_account(**self.args)[1] File "/path/pkg/shade/openstackcloud.py", line 849, in swift_client 'object-store', swiftclient.client.Connection) File "/path/pkg/shade/openstackcloud.py", line 343, in _get_client **kwargs) File "/path/pkg/os_client_config/cloud_config.py", line 301, in get_legacy_client return self._get_swift_client(client_class=client_class, **kwargs) File "/path/pkg/os_client_config/cloud_config.py", line 369, in _get_swift_client endpoint = self.get_session_endpoint(service_key='object-store') File "/path/pkg/os_client_config/cloud_config.py", line 253, in get_session_endpoint region_name=self.region) File "/path/pkg/keystoneauth1/session.py", line 765, in get_endpoint return auth.get_endpoint(self, **kwargs) File "/path/pkg/keystoneauth1/identity/base.py", line 216, in get_endpoint service_name=service_name) File "/path/pkg/positional/__init__.py", line 101, in inner return wrapped(*args, **kwargs) File "/path/pkg/keystoneauth1/access/service_catalog.py", line 228, in url_for raise exceptions.EndpointNotFound(msg) keystoneauth1.exceptions.catalog.EndpointNotFound: public endpoint for object-store service in RegionOne region not found Change-Id: Idbf5081117bb0a13d04a1a5cb9fd7682baaf04e5 --- os_client_config/cloud_config.py | 18 +++++++++++++----- os_client_config/tests/test_cloud_config.py | 9 +++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index ae5da3a5a..a911d8102 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -16,6 +16,7 @@ import importlib import warnings from keystoneauth1 import adapter +import keystoneauth1.exceptions.catalog from keystoneauth1 import plugin from keystoneauth1 import session import requestsexceptions @@ -247,11 +248,18 @@ class CloudConfig(object): if service_key == 'identity': endpoint = session.get_endpoint(interface=plugin.AUTH_INTERFACE) else: - endpoint = session.get_endpoint( - service_type=self.get_service_type(service_key), - service_name=self.get_service_name(service_key), - interface=self.get_interface(service_key), - region_name=self.region) + args = { + 'service_type': self.get_service_type(service_key), + 'service_name': self.get_service_name(service_key), + 'interface': self.get_interface(service_key), + 'region_name': self.region + } + try: + endpoint = session.get_endpoint(**args) + except keystoneauth1.exceptions.catalog.EndpointNotFound: + self.log.warning("Keystone catalog entry not found (%s)", + args) + endpoint = None return endpoint def get_legacy_client( diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 7a8b77aed..2763f4d3d 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -12,6 +12,7 @@ import copy +from keystoneauth1 import exceptions as ksa_exceptions from keystoneauth1 import plugin as ksa_plugin from keystoneauth1 import session as ksa_session import mock @@ -235,6 +236,14 @@ class TestCloudConfig(base.TestCase): region_name='region-al', service_type='orchestration') + @mock.patch.object(cloud_config.CloudConfig, 'get_session') + def test_session_endpoint_not_found(self, mock_get_session): + exc_to_raise = ksa_exceptions.catalog.EndpointNotFound + mock_get_session.return_value.get_endpoint.side_effect = exc_to_raise + cc = cloud_config.CloudConfig( + "test1", "region-al", {}, auth_plugin=mock.Mock()) + self.assertIsNone(cc.get_session_endpoint('notfound')) + @mock.patch.object(cloud_config.CloudConfig, 'get_api_version') @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') From 8a8a218f090ea7bbb7ce1d3afc11d8f2fda06873 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Thu, 6 Oct 2016 20:41:15 +0200 Subject: [PATCH 299/365] Enable release notes translation Releasenote translation publishing is being prepared. 'locale_dirs' needs to be defined in conf.py to generate translated version of the release notes. Note that this repository might not get translated release notes - or no translations at all - but we add the entry here nevertheless to prepare for it. Change-Id: Ic34d4d11adf4aacd91a7fd682a6d15597004ff49 --- releasenotes/source/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index 282dbd784..e33ee8e57 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -259,3 +259,6 @@ texinfo_documents = [ # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] From 1653120803e76cf02c2f35d3bdb67e6bb1848817 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 7 Oct 2016 11:35:50 -0400 Subject: [PATCH 300/365] Add setter for session constructor shade needs to be able to attach an adapter wrapper to an already constructed CloudConfig object, so add a setter. Change-Id: I640859b5d78d17e3c99e8ec11f1418f275e4dea2 --- os_client_config/cloud_config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index ae5da3a5a..80cd266ec 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -102,6 +102,10 @@ class CloudConfig(object): def __ne__(self, other): return not self == other + def set_session_constructor(self, session_constructor): + """Sets the Session constructor.""" + self._session_constructor = session_constructor + def get_requests_verify_args(self): """Return the verify and cert values for the requests library.""" if self.config['verify'] and self.config['cacert']: From 6623be208dc009cd1667c7d5db0b9df89ea3a80f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 12 Oct 2016 15:05:02 -0500 Subject: [PATCH 301/365] Revert "Split auth plugin loading into its own method" This reverts commit 8b7859e21e64027d20f158737bbf70bbe409b847. python-openstackclient has a subclass that defines this method with a different signature. Change-Id: Ie44f8efb6b93dc0d4754fb316ddb9087ce181275 --- os_client_config/config.py | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 01f659e49..5ab0d907e 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -1035,24 +1035,6 @@ class OpenStackConfig(object): return config - def load_auth_plugin(self, config, cloud): - try: - loader = self._get_auth_loader(config) - config = self._validate_auth(config, loader) - auth_plugin = loader.load_from_options(**config['auth']) - except Exception as e: - # We WANT the ksa exception normally - # but OSC can't handle it right now, so we try deferring - # to ksc. If that ALSO fails, it means there is likely - # a deeper issue, so we assume the ksa error was correct - self.log.debug("Deferring keystone exception: {e}".format(e=e)) - auth_plugin = None - try: - config = self._validate_auth_ksc(config, cloud) - except Exception: - raise e - return auth_plugin - def get_one_cloud(self, cloud=None, validate=True, argparse=None, **kwargs): """Retrieve a single cloud configuration and merge additional options @@ -1111,7 +1093,21 @@ class OpenStackConfig(object): config = self.auth_config_hook(config) if validate: - auth_plugin = self.load_auth_plugin(config, cloud) + try: + loader = self._get_auth_loader(config) + config = self._validate_auth(config, loader) + auth_plugin = loader.load_from_options(**config['auth']) + except Exception as e: + # We WANT the ksa exception normally + # but OSC can't handle it right now, so we try deferring + # to ksc. If that ALSO fails, it means there is likely + # a deeper issue, so we assume the ksa error was correct + self.log.debug("Deferring keystone exception: {e}".format(e=e)) + auth_plugin = None + try: + config = self._validate_auth_ksc(config, cloud) + except Exception: + raise e else: auth_plugin = None From 47068d0abbb1018e238213af30cf7840bb2104d9 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 19 Oct 2016 16:17:38 -0500 Subject: [PATCH 302/365] Update ECS image_api_version to 1 Turns out they don't run v2. Change-Id: Icd503f2b035400fbb39903b3fe2542ec14b86e93 --- doc/source/vendor-support.rst | 1 + os_client_config/vendors/entercloudsuite.json | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index cfdc02eeb..eb7ecf0fe 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -160,6 +160,7 @@ it-mil1 Milan, IT de-fra1 Frankfurt, DE ============== ================ +* Image API Version is 1 * Volume API Version is 1 internap diff --git a/os_client_config/vendors/entercloudsuite.json b/os_client_config/vendors/entercloudsuite.json index 6d2fc129e..c58c478f0 100644 --- a/os_client_config/vendors/entercloudsuite.json +++ b/os_client_config/vendors/entercloudsuite.json @@ -5,6 +5,7 @@ "auth_url": "https://api.entercloudsuite.com/" }, "identity_api_version": "3", + "image_api_version": "1", "volume_api_version": "1", "regions": [ "it-mil1", From 422ad9ccdb8b08a223af78a1d1f48df703dd1a9d Mon Sep 17 00:00:00 2001 From: Cedric Brandily Date: Tue, 18 Oct 2016 22:20:00 +0200 Subject: [PATCH 303/365] Clarify how to set SSL settings This change adds an example in order to clarif how to SSL settings in configuration files. Change-Id: Id047f21d0a51752f46b16e3f4efbfec62cfbd5fb --- README.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.rst b/README.rst index a47e98bed..f3d41c590 100644 --- a/README.rst +++ b/README.rst @@ -185,6 +185,19 @@ Client certs are also configurable. `cert` will be the client cert file location. In case the cert key is not included within the client cert file, its file location needs to be set via `key`. +.. code-block:: yaml + + # clouds.yaml + clouds: + secure: + auth: ... + key: /home/myhome/client-cert.key + cert: /home/myhome/client-cert.crt + cacert: /home/myhome/ca.crt + insecure: + auth: ... + verify: False + Cache Settings -------------- From 1ac6766537f6ac8f2b1c22ac726a2fb8ad9ce6dd Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 8 Sep 2016 16:10:16 -0500 Subject: [PATCH 304/365] Fix a bunch of tests There is a bug in validate_auth that gets hidden by validate_auth_ksc. These tests are the test fixes for it. Change-Id: I80d558a1c794725ba2a87fbd87bf8fbdf6633bee --- os_client_config/config.py | 9 +++++++++ os_client_config/tests/test_config.py | 6 +++--- os_client_config/tests/test_environ.py | 28 +++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 6ff2359ff..a0d862d15 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -935,6 +935,15 @@ class OpenStackConfig(object): winning_value, ) + if winning_value: + # Prefer the plugin configuration dest value if the value's key + # is marked as deprecated. + if p_opt.dest is None: + good_name = p_opt.name.replace('-', '_') + config['auth'][good_name] = winning_value + else: + config['auth'][p_opt.dest] = winning_value + # See if this needs a prompting config = self.option_prompt(config, p_opt) diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index 0ef8da2f2..ad3685ab1 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -735,7 +735,7 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(opts.http_timeout, '20') with testtools.ExpectedException(AttributeError): opts.os_network_service_type - cloud = c.get_one_cloud(argparse=opts, verify=False) + cloud = c.get_one_cloud(argparse=opts, validate=False) self.assertEqual(cloud.config['service_type'], 'network') self.assertEqual(cloud.config['interface'], 'admin') self.assertEqual(cloud.config['api_timeout'], '20') @@ -756,7 +756,7 @@ class TestConfigArgparse(base.TestCase): self.assertIsNone(opts.os_network_service_type) self.assertIsNone(opts.os_network_api_version) self.assertEqual(opts.network_api_version, '4') - cloud = c.get_one_cloud(argparse=opts, verify=False) + cloud = c.get_one_cloud(argparse=opts, validate=False) self.assertEqual(cloud.config['service_type'], 'network') self.assertEqual(cloud.config['interface'], 'admin') self.assertEqual(cloud.config['network_api_version'], '4') @@ -783,7 +783,7 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(opts.os_endpoint_type, 'admin') self.assertIsNone(opts.os_network_api_version) self.assertEqual(opts.network_api_version, '4') - cloud = c.get_one_cloud(argparse=opts, verify=False) + cloud = c.get_one_cloud(argparse=opts, validate=False) self.assertEqual(cloud.config['service_type'], 'compute') self.assertEqual(cloud.config['network_service_type'], 'badtype') self.assertEqual(cloud.config['interface'], 'admin') diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index b75db1c61..35ce2f2bf 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -139,11 +139,30 @@ class TestEnvvars(base.TestCase): self.assertRaises( exceptions.OpenStackConfigException, c.get_one_cloud, 'envvars') - def test_have_envvars(self): + def test_incomplete_envvars(self): self.useFixture( fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) self.useFixture( fixtures.EnvironmentVariable('OS_USERNAME', 'user')) + config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + # This is broken due to an issue that's fixed in a subsequent patch + # commenting it out in this patch to keep the patch size reasonable + # self.assertRaises( + # keystoneauth1.exceptions.auth_plugins.MissingRequiredOptions, + # c.get_one_cloud, 'envvars') + + def test_have_envvars(self): + self.useFixture( + fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) + self.useFixture( + fixtures.EnvironmentVariable('OS_AUTH_URL', 'http://example.com')) + self.useFixture( + fixtures.EnvironmentVariable('OS_USERNAME', 'user')) + self.useFixture( + fixtures.EnvironmentVariable('OS_PASSWORD', 'password')) + self.useFixture( + fixtures.EnvironmentVariable('OS_PROJECT_NAME', 'project')) c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud('envvars') @@ -152,6 +171,13 @@ class TestEnvvars(base.TestCase): def test_old_envvars(self): self.useFixture( fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) + self.useFixture( + fixtures.EnvironmentVariable( + 'NOVA_AUTH_URL', 'http://example.com')) + self.useFixture( + fixtures.EnvironmentVariable('NOVA_PASSWORD', 'password')) + self.useFixture( + fixtures.EnvironmentVariable('NOVA_PROJECT_NAME', 'project')) c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml], envvar_prefix='NOVA_') From 86ade8f5281d2bb8b7c1637436aa8cb0c7cddb98 Mon Sep 17 00:00:00 2001 From: Ghe Rivero Date: Fri, 21 Oct 2016 12:16:51 +0200 Subject: [PATCH 305/365] Normalize cloud config before osc-lib call Cloud config was passed to osc-lib before being normalized, causing exceptions when some api versions were stil an int where osc-lib expects a str Change-Id: I7326114d86a4208f1489c302e8bb838dd5b8c5d6 --- os_client_config/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 6ff2359ff..774a2599e 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -1085,6 +1085,7 @@ class OpenStackConfig(object): config[key] = val config = self.magic_fixes(config) + config = self._normalize_keys(config) # NOTE(dtroyer): OSC needs a hook into the auth args before the # plugin is loaded in order to maintain backward- @@ -1127,7 +1128,7 @@ class OpenStackConfig(object): return cloud_config.CloudConfig( name=cloud_name, region=config['region_name'], - config=self._normalize_keys(config), + config=config, force_ipv4=force_ipv4, auth_plugin=auth_plugin, openstack_config=self, From 3f525e01798b433732139a9bff88d9360613cd71 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 20 Oct 2016 08:41:18 -0500 Subject: [PATCH 306/365] Add support for volumev3 service type Words cannot begin to adequately express the disappointment and rage I felt upon learning that the cinder team had unleashed 'volumev3' upon the world. Woe betide us, the mere users, for wanting to use a 'volume' endpoint and have that choice mean something. Perhaps if we beat ourselves with leather straps while crawling for days across the country we can remove more joy from our lives. In the meantime, until we can fully appreciate the existential crisis of being, let's continue to work around it in os-client-config. Change-Id: I171e3b01497b3e3a06c3a73577f0f67e0c1e6f73 --- os_client_config/cloud_config.py | 9 ++++++--- os_client_config/tests/test_cloud_config.py | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 147eb04d4..82442bb78 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -160,9 +160,12 @@ class CloudConfig(object): # atrocity from the as-yet-unsullied eyes of our users. # Of course, if the user requests a volumev2, that structure should # still work. - if (service_type == 'volume' and - self.get_api_version(service_type).startswith('2')): - service_type = 'volumev2' + # What's even more amazing is that they did it AGAIN with cinder v3 + if service_type == 'volume': + if self.get_api_version(service_type).startswith('2'): + service_type = 'volumev2' + elif self.get_api_version(service_type).startswith('3'): + service_type = 'volumev3' return self.config.get(key, service_type) def get_service_name(self, service_type): diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 2763f4d3d..e3d1d5d6f 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -158,6 +158,11 @@ class TestCloudConfig(base.TestCase): cc.config['volume_api_version'] = '2' self.assertEqual('volumev2', cc.get_service_type('volume')) + def test_volume_override_v3(self): + cc = cloud_config.CloudConfig("test1", "region-al", fake_services_dict) + cc.config['volume_api_version'] = '3' + self.assertEqual('volumev3', cc.get_service_type('volume')) + def test_get_session_no_auth(self): config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) From b5d65b74f60ce743b03b49a6c176700d658cfe98 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 24 Oct 2016 05:40:32 +0200 Subject: [PATCH 307/365] Support token_endpoint as an auth_type For backwards compat with what operators have been trained to do, map token_endpoint to admin_token for them. This has shown up a few times in the wild. Most recently: https://github.com/ansible/ansible-modules-core/issues/5250 Change-Id: Ie083381e7fda19e016b6425939bd3c2dc260fa9b --- os_client_config/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index a0d862d15..dc3decd38 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -854,6 +854,11 @@ class OpenStackConfig(object): # Set to notused rather than None because validate_auth will # strip the value if it's actually python None config['auth']['token'] = 'notused' + elif config['auth_type'] == 'token_endpoint': + # Humans have been trained to use a thing called token_endpoint + # That it does not exist in keystoneauth is irrelvant- it not + # doing what they want causes them sorrow. + config['auth_type'] = 'admin_token' return loading.get_plugin_loader(config['auth_type']) def _validate_auth_ksc(self, config, cloud): From 9f47acc0cd0c69888b1a3d98b98fcf1a88c248a5 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 5 Nov 2016 10:26:32 -0500 Subject: [PATCH 308/365] Add fuga.io to vendors Change-Id: I1704b6e26a3c56b519544ad8ee6d3fd80a2a752a --- doc/source/vendor-support.rst | 14 ++++++++++++++ os_client_config/vendors/fuga.json | 15 +++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 os_client_config/vendors/fuga.json diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index eb7ecf0fe..b301d8017 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -163,6 +163,20 @@ de-fra1 Frankfurt, DE * Image API Version is 1 * Volume API Version is 1 +fuga +---- + +https://identity.api.fuga.io:5000 + +============== ================ +Region Name Location +============== ================ +cystack Netherlands +============== ================ + +* Identity API Version is 3 +* Volume API Version is 3 + internap -------- diff --git a/os_client_config/vendors/fuga.json b/os_client_config/vendors/fuga.json new file mode 100644 index 000000000..388500b1b --- /dev/null +++ b/os_client_config/vendors/fuga.json @@ -0,0 +1,15 @@ +{ + "name": "fuga", + "profile": { + "auth": { + "auth_url": "https://identity.api.fuga.io:5000", + "user_domain_name": "Default", + "project_domain_name": "Default" + }, + "regions": [ + "cystack" + ], + "identity_api_version": "3", + "volume_api_version": "3" + } +} From 1f9e2cd123b38a7e744fb8a784d0ee3b523de95e Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 1 Sep 2016 13:26:00 -0500 Subject: [PATCH 309/365] Remove validate_auth_ksc This was a workaround for python-openstackclient back when it was still dependent on keystoneclient. OSC has its own workaround now, so this should no longer be needed. Change-Id: Ib1877b7978b7b016b394232235e887360b6bdf85 --- os_client_config/config.py | 71 ++++---------------------- os_client_config/tests/test_config.py | 28 ++++++++-- os_client_config/tests/test_environ.py | 17 +++--- os_client_config/tests/test_init.py | 5 +- 4 files changed, 45 insertions(+), 76 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index a0d862d15..6078366e6 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -609,15 +609,19 @@ class OpenStackConfig(object): 'tenant_id', 'tenant-id', 'project_id', 'project-id') mappings['project_name'] = ( 'tenant_name', 'tenant-name', 'project_name', 'project-name') + # Special-case username and password so that we don't have to load + # the plugins so early + mappings['username'] = ('user-name', 'user_name', 'username') + mappings['password'] = ('password',) for target_key, possible_values in mappings.items(): target = None for key in possible_values: - if key in cloud: - target = str(cloud[key]) - del cloud[key] - if key in cloud['auth']: - target = str(cloud['auth'][key]) - del cloud['auth'][key] + root_target = cloud.pop(key, None) + auth_target = cloud['auth'].pop(key, None) + if root_target: + target = str(root_target) + elif auth_target: + target = str(auth_target) if target: cloud['auth'][target_key] = target return cloud @@ -856,59 +860,6 @@ class OpenStackConfig(object): config['auth']['token'] = 'notused' return loading.get_plugin_loader(config['auth_type']) - def _validate_auth_ksc(self, config, cloud): - try: - import keystoneclient.auth as ksc_auth - except ImportError: - return config - - # May throw a keystoneclient.exceptions.NoMatchingPlugin - plugin_options = ksc_auth.get_plugin_class( - config['auth_type']).get_options() - - for p_opt in plugin_options: - # if it's in config.auth, win, kill it from config dict - # if it's in config and not in config.auth, move it - # deprecated loses to current - # provided beats default, deprecated or not - winning_value = self._find_winning_auth_value( - p_opt, - config['auth'], - ) - if not winning_value: - winning_value = self._find_winning_auth_value( - p_opt, - config, - ) - - # if the plugin tells us that this value is required - # then error if it's doesn't exist now - if not winning_value and p_opt.required: - raise exceptions.OpenStackConfigException( - 'Unable to find auth information for cloud' - ' {cloud} in config files {files}' - ' or environment variables. Missing value {auth_key}' - ' required for auth plugin {plugin}'.format( - cloud=cloud, files=','.join(self._config_files), - auth_key=p_opt.name, plugin=config.get('auth_type'))) - - # Clean up after ourselves - for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: - opt = opt.replace('-', '_') - config.pop(opt, None) - config['auth'].pop(opt, None) - - if winning_value: - # Prefer the plugin configuration dest value if the value's key - # is marked as depreciated. - if p_opt.dest is None: - config['auth'][p_opt.name.replace('-', '_')] = ( - winning_value) - else: - config['auth'][p_opt.dest] = winning_value - - return config - def _validate_auth(self, config, loader): # May throw a keystoneauth1.exceptions.NoMatchingPlugin @@ -1016,6 +967,7 @@ class OpenStackConfig(object): ('auth_token' in config and config['auth_token']) or ('token' in config and config['token'])): config.setdefault('token', config.pop('auth_token', None)) + config.setdefault('auth_type', 'token') # These backwards compat values are only set via argparse. If it's # there, it's because it was passed in explicitly, and should win @@ -1062,7 +1014,6 @@ class OpenStackConfig(object): :raises: keystoneauth1.exceptions.MissingRequiredOptions on missing required auth parameters """ - args = self._fix_args(kwargs, argparse=argparse) if cloud is None: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index ad3685ab1..efab79792 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -226,7 +226,7 @@ class TestConfig(base.TestCase): c = config.OpenStackConfig(config_files=['nonexistent'], vendor_files=['nonexistent'], secure_files=[self.secure_yaml]) - cc = c.get_one_cloud(cloud='_test_cloud_no_vendor') + cc = c.get_one_cloud(cloud='_test_cloud_no_vendor', validate=False) self.assertEqual('testpass', cc.auth['password']) def test_get_cloud_names(self): @@ -366,7 +366,6 @@ class TestConfigArgparse(base.TestCase): project_name='project', region_name='region2', snack_type='cookie', - os_auth_token='no-good-things', ) self.options = argparse.Namespace(**self.args) @@ -417,7 +416,7 @@ class TestConfigArgparse(base.TestCase): cc = c.get_one_cloud( argparse=options, **kwargs) self.assertEqual(cc.region_name, 'region2') - self.assertEqual(cc.auth['password'], 'authpass') + self.assertEqual(cc.auth['password'], 'argpass') self.assertEqual(cc.snack_type, 'cookie') def test_get_one_cloud_precedence_osc(self): @@ -474,7 +473,7 @@ class TestConfigArgparse(base.TestCase): cc = c.get_one_cloud(**kwargs) self.assertEqual(cc.region_name, 'kwarg_region') - self.assertEqual(cc.auth['password'], 'authpass') + self.assertEqual(cc.auth['password'], 'ansible_password') self.assertIsNone(cc.password) def test_get_one_cloud_just_argparse(self): @@ -649,11 +648,30 @@ class TestConfigArgparse(base.TestCase): parser.add_argument('--os-auth-token') opts, _remain = parser.parse_known_args( ['--os-auth-token', 'very-bad-things', - '--os-auth-type', 'token']) + '--os-auth-type', 'token', + '--os-auth-url', 'http://example.com/v2', + '--os-project-name', 'project']) cc = c.get_one_cloud(argparse=opts) self.assertEqual(cc.config['auth_type'], 'token') self.assertEqual(cc.config['auth']['token'], 'very-bad-things') + def test_argparse_username_token(self): + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + + parser = argparse.ArgumentParser() + c.register_argparse_arguments(parser, []) + # novaclient will add this + parser.add_argument('--os-auth-token') + opts, _remain = parser.parse_known_args( + ['--os-auth-token', 'very-bad-things', + '--os-auth-type', 'token', + '--os-auth-url', 'http://example.com/v2', + '--os-username', 'user', + '--os-project-name', 'project']) + self.assertRaises( + TypeError, c.get_one_cloud, argparse=opts) + def test_argparse_underscores(self): c = config.OpenStackConfig(config_files=[self.no_yaml], vendor_files=[self.no_yaml], diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 35ce2f2bf..9cece4887 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -19,6 +19,7 @@ from os_client_config import exceptions from os_client_config.tests import base import fixtures +import keystoneauth1.exceptions class TestEnviron(base.TestCase): @@ -144,13 +145,11 @@ class TestEnvvars(base.TestCase): fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) self.useFixture( fixtures.EnvironmentVariable('OS_USERNAME', 'user')) - config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) - # This is broken due to an issue that's fixed in a subsequent patch - # commenting it out in this patch to keep the patch size reasonable - # self.assertRaises( - # keystoneauth1.exceptions.auth_plugins.MissingRequiredOptions, - # c.get_one_cloud, 'envvars') + c = config.OpenStackConfig( + config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) + self.assertRaises( + keystoneauth1.exceptions.auth_plugins.MissingRequiredOptions, + c.get_one_cloud, 'envvars') def test_have_envvars(self): self.useFixture( @@ -165,7 +164,7 @@ class TestEnvvars(base.TestCase): fixtures.EnvironmentVariable('OS_PROJECT_NAME', 'project')) c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud('envvars') + cc = c.get_one_cloud('envvars', validate=False) self.assertEqual(cc.config['auth']['username'], 'user') def test_old_envvars(self): @@ -181,5 +180,5 @@ class TestEnvvars(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml], envvar_prefix='NOVA_') - cc = c.get_one_cloud('envvars') + cc = c.get_one_cloud('envvars', validate=False) self.assertEqual(cc.config['auth']['username'], 'nova') diff --git a/os_client_config/tests/test_init.py b/os_client_config/tests/test_init.py index 15d57f717..76ad48597 100644 --- a/os_client_config/tests/test_init.py +++ b/os_client_config/tests/test_init.py @@ -18,7 +18,8 @@ from os_client_config.tests import base class TestInit(base.TestCase): def test_get_config_without_arg_parser(self): - cloud_config = os_client_config.get_config(options=None) + cloud_config = os_client_config.get_config( + options=None, validate=False) self.assertIsInstance( cloud_config, os_client_config.cloud_config.CloudConfig @@ -26,7 +27,7 @@ class TestInit(base.TestCase): def test_get_config_with_arg_parser(self): cloud_config = os_client_config.get_config( - options=argparse.ArgumentParser()) + options=argparse.ArgumentParser(), validate=False) self.assertIsInstance( cloud_config, os_client_config.cloud_config.CloudConfig From e2a593d917533424c6de39774afb8566d4f81db2 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 14 Nov 2016 13:22:49 -0600 Subject: [PATCH 310/365] Revert "Remove validate_auth_ksc" This reverts commit 1f9e2cd123b38a7e744fb8a784d0ee3b523de95e. Sad as this makes me, let's revert and come back to it when we figure out the cliff thing. Change-Id: I0413d5e3b3d8652833a8e7942ba81926787ba3bf --- os_client_config/config.py | 71 ++++++++++++++++++++++---- os_client_config/tests/test_config.py | 28 ++-------- os_client_config/tests/test_environ.py | 17 +++--- os_client_config/tests/test_init.py | 5 +- 4 files changed, 76 insertions(+), 45 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 22bd00c7a..9b4a709fd 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -609,19 +609,15 @@ class OpenStackConfig(object): 'tenant_id', 'tenant-id', 'project_id', 'project-id') mappings['project_name'] = ( 'tenant_name', 'tenant-name', 'project_name', 'project-name') - # Special-case username and password so that we don't have to load - # the plugins so early - mappings['username'] = ('user-name', 'user_name', 'username') - mappings['password'] = ('password',) for target_key, possible_values in mappings.items(): target = None for key in possible_values: - root_target = cloud.pop(key, None) - auth_target = cloud['auth'].pop(key, None) - if root_target: - target = str(root_target) - elif auth_target: - target = str(auth_target) + if key in cloud: + target = str(cloud[key]) + del cloud[key] + if key in cloud['auth']: + target = str(cloud['auth'][key]) + del cloud['auth'][key] if target: cloud['auth'][target_key] = target return cloud @@ -865,6 +861,59 @@ class OpenStackConfig(object): config['auth_type'] = 'admin_token' return loading.get_plugin_loader(config['auth_type']) + def _validate_auth_ksc(self, config, cloud): + try: + import keystoneclient.auth as ksc_auth + except ImportError: + return config + + # May throw a keystoneclient.exceptions.NoMatchingPlugin + plugin_options = ksc_auth.get_plugin_class( + config['auth_type']).get_options() + + for p_opt in plugin_options: + # if it's in config.auth, win, kill it from config dict + # if it's in config and not in config.auth, move it + # deprecated loses to current + # provided beats default, deprecated or not + winning_value = self._find_winning_auth_value( + p_opt, + config['auth'], + ) + if not winning_value: + winning_value = self._find_winning_auth_value( + p_opt, + config, + ) + + # if the plugin tells us that this value is required + # then error if it's doesn't exist now + if not winning_value and p_opt.required: + raise exceptions.OpenStackConfigException( + 'Unable to find auth information for cloud' + ' {cloud} in config files {files}' + ' or environment variables. Missing value {auth_key}' + ' required for auth plugin {plugin}'.format( + cloud=cloud, files=','.join(self._config_files), + auth_key=p_opt.name, plugin=config.get('auth_type'))) + + # Clean up after ourselves + for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: + opt = opt.replace('-', '_') + config.pop(opt, None) + config['auth'].pop(opt, None) + + if winning_value: + # Prefer the plugin configuration dest value if the value's key + # is marked as depreciated. + if p_opt.dest is None: + config['auth'][p_opt.name.replace('-', '_')] = ( + winning_value) + else: + config['auth'][p_opt.dest] = winning_value + + return config + def _validate_auth(self, config, loader): # May throw a keystoneauth1.exceptions.NoMatchingPlugin @@ -972,7 +1021,6 @@ class OpenStackConfig(object): ('auth_token' in config and config['auth_token']) or ('token' in config and config['token'])): config.setdefault('token', config.pop('auth_token', None)) - config.setdefault('auth_type', 'token') # These backwards compat values are only set via argparse. If it's # there, it's because it was passed in explicitly, and should win @@ -1019,6 +1067,7 @@ class OpenStackConfig(object): :raises: keystoneauth1.exceptions.MissingRequiredOptions on missing required auth parameters """ + args = self._fix_args(kwargs, argparse=argparse) if cloud is None: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index efab79792..ad3685ab1 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -226,7 +226,7 @@ class TestConfig(base.TestCase): c = config.OpenStackConfig(config_files=['nonexistent'], vendor_files=['nonexistent'], secure_files=[self.secure_yaml]) - cc = c.get_one_cloud(cloud='_test_cloud_no_vendor', validate=False) + cc = c.get_one_cloud(cloud='_test_cloud_no_vendor') self.assertEqual('testpass', cc.auth['password']) def test_get_cloud_names(self): @@ -366,6 +366,7 @@ class TestConfigArgparse(base.TestCase): project_name='project', region_name='region2', snack_type='cookie', + os_auth_token='no-good-things', ) self.options = argparse.Namespace(**self.args) @@ -416,7 +417,7 @@ class TestConfigArgparse(base.TestCase): cc = c.get_one_cloud( argparse=options, **kwargs) self.assertEqual(cc.region_name, 'region2') - self.assertEqual(cc.auth['password'], 'argpass') + self.assertEqual(cc.auth['password'], 'authpass') self.assertEqual(cc.snack_type, 'cookie') def test_get_one_cloud_precedence_osc(self): @@ -473,7 +474,7 @@ class TestConfigArgparse(base.TestCase): cc = c.get_one_cloud(**kwargs) self.assertEqual(cc.region_name, 'kwarg_region') - self.assertEqual(cc.auth['password'], 'ansible_password') + self.assertEqual(cc.auth['password'], 'authpass') self.assertIsNone(cc.password) def test_get_one_cloud_just_argparse(self): @@ -648,30 +649,11 @@ class TestConfigArgparse(base.TestCase): parser.add_argument('--os-auth-token') opts, _remain = parser.parse_known_args( ['--os-auth-token', 'very-bad-things', - '--os-auth-type', 'token', - '--os-auth-url', 'http://example.com/v2', - '--os-project-name', 'project']) + '--os-auth-type', 'token']) cc = c.get_one_cloud(argparse=opts) self.assertEqual(cc.config['auth_type'], 'token') self.assertEqual(cc.config['auth']['token'], 'very-bad-things') - def test_argparse_username_token(self): - c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) - - parser = argparse.ArgumentParser() - c.register_argparse_arguments(parser, []) - # novaclient will add this - parser.add_argument('--os-auth-token') - opts, _remain = parser.parse_known_args( - ['--os-auth-token', 'very-bad-things', - '--os-auth-type', 'token', - '--os-auth-url', 'http://example.com/v2', - '--os-username', 'user', - '--os-project-name', 'project']) - self.assertRaises( - TypeError, c.get_one_cloud, argparse=opts) - def test_argparse_underscores(self): c = config.OpenStackConfig(config_files=[self.no_yaml], vendor_files=[self.no_yaml], diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 9cece4887..35ce2f2bf 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -19,7 +19,6 @@ from os_client_config import exceptions from os_client_config.tests import base import fixtures -import keystoneauth1.exceptions class TestEnviron(base.TestCase): @@ -145,11 +144,13 @@ class TestEnvvars(base.TestCase): fixtures.EnvironmentVariable('NOVA_USERNAME', 'nova')) self.useFixture( fixtures.EnvironmentVariable('OS_USERNAME', 'user')) - c = config.OpenStackConfig( - config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - self.assertRaises( - keystoneauth1.exceptions.auth_plugins.MissingRequiredOptions, - c.get_one_cloud, 'envvars') + config.OpenStackConfig(config_files=[self.cloud_yaml], + vendor_files=[self.vendor_yaml]) + # This is broken due to an issue that's fixed in a subsequent patch + # commenting it out in this patch to keep the patch size reasonable + # self.assertRaises( + # keystoneauth1.exceptions.auth_plugins.MissingRequiredOptions, + # c.get_one_cloud, 'envvars') def test_have_envvars(self): self.useFixture( @@ -164,7 +165,7 @@ class TestEnvvars(base.TestCase): fixtures.EnvironmentVariable('OS_PROJECT_NAME', 'project')) c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud('envvars', validate=False) + cc = c.get_one_cloud('envvars') self.assertEqual(cc.config['auth']['username'], 'user') def test_old_envvars(self): @@ -180,5 +181,5 @@ class TestEnvvars(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml], envvar_prefix='NOVA_') - cc = c.get_one_cloud('envvars', validate=False) + cc = c.get_one_cloud('envvars') self.assertEqual(cc.config['auth']['username'], 'nova') diff --git a/os_client_config/tests/test_init.py b/os_client_config/tests/test_init.py index 76ad48597..15d57f717 100644 --- a/os_client_config/tests/test_init.py +++ b/os_client_config/tests/test_init.py @@ -18,8 +18,7 @@ from os_client_config.tests import base class TestInit(base.TestCase): def test_get_config_without_arg_parser(self): - cloud_config = os_client_config.get_config( - options=None, validate=False) + cloud_config = os_client_config.get_config(options=None) self.assertIsInstance( cloud_config, os_client_config.cloud_config.CloudConfig @@ -27,7 +26,7 @@ class TestInit(base.TestCase): def test_get_config_with_arg_parser(self): cloud_config = os_client_config.get_config( - options=argparse.ArgumentParser(), validate=False) + options=argparse.ArgumentParser()) self.assertIsInstance( cloud_config, os_client_config.cloud_config.CloudConfig From 3008683d614b62a87057d6dd510716291ba407cb Mon Sep 17 00:00:00 2001 From: Flavio Percoco Date: Fri, 25 Nov 2016 10:55:46 +0100 Subject: [PATCH 311/365] Show team and repo badges on README This patch adds the team's and repository's badges to the README file. The motivation behind this is to communicate the project status and features at first glance. For more information about this effort, please read this email thread: http://lists.openstack.org/pipermail/openstack-dev/2016-October/105562.html To see an example of how this would look like check: https://gist.github.com/0bed66f234ed59d0c6e2aa220e709950 Change-Id: I80c57e310b601d47be5b5f81c604d7d6310a60eb --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index f3d41c590..9e460be85 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,12 @@ +======================== +Team and repository tags +======================== + +.. image:: http://governance.openstack.org/badges/os-client-config.svg + :target: http://governance.openstack.org/reference/tags/index.html + +.. Change things from this point on + ================ os-client-config ================ From fa4e1bd21db4bee2a0ee779067cdd659e647d7fc Mon Sep 17 00:00:00 2001 From: Paulo Matias Date: Wed, 7 Dec 2016 18:07:37 -0200 Subject: [PATCH 312/365] Fix interface_key for identity clients Change-Id: I83870e8b3ee6dc7fdbb6e9d67075cc4c08646e4e Closes-Bug: #1648212 --- os_client_config/cloud_config.py | 2 +- os_client_config/tests/test_cloud_config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 82442bb78..f61c49814 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -321,7 +321,7 @@ class CloudConfig(object): endpoint_override = self.get_endpoint(service_key) if not interface_key: - if service_key in ('image', 'key-manager'): + if service_key in ('image', 'key-manager', 'identity'): interface_key = 'interface' else: interface_key = 'endpoint_type' diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index e3d1d5d6f..dcbeb3a55 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -590,7 +590,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='2.0', endpoint='http://example.com/v2', - endpoint_type='admin', + interface='admin', endpoint_override=None, region_name='region-al', service_type='identity', @@ -610,7 +610,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='3', endpoint='http://example.com', - endpoint_type='admin', + interface='admin', endpoint_override=None, region_name='region-al', service_type='identity', From e2d1008aed7f0d3046e1aa539c03837318c4263a Mon Sep 17 00:00:00 2001 From: Steve Martinelli Date: Thu, 15 Dec 2016 12:34:32 -0500 Subject: [PATCH 313/365] Add docutils contraint on 0.13.1 to fix building See: http://lists.openstack.org/pipermail/openstack-dev/2016-December/108742.html Change-Id: Id183b0d3a6339e10f2f1f4a5dc78352d5c3f27ed --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index 0138f13f6..f9908d6e0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,6 +5,7 @@ hacking>=0.10.2,<0.11 # Apache-2.0 coverage>=3.6 +docutils>=0.11,!=0.13.1 # OSI-Approved Open Source, Public Domain extras fixtures>=0.3.14 jsonschema>=2.0.0,<3.0.0,!=2.5.0 From 6615160361f375f480ce66947957513ee3dd0134 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 15 Dec 2016 10:07:55 -0600 Subject: [PATCH 314/365] Magnum's service_type is container_infra Leave container_api_version since people might be depending on it. Also, add it to the constructors list, since we do know about it. Change-Id: I3bcb966154ac53269614c943ad9c2675b27d62d0 --- os_client_config/constructors.json | 1 + os_client_config/defaults.json | 1 + 2 files changed, 2 insertions(+) diff --git a/os_client_config/constructors.json b/os_client_config/constructors.json index 89c844c55..a78be6bcb 100644 --- a/os_client_config/constructors.json +++ b/os_client_config/constructors.json @@ -1,5 +1,6 @@ { "compute": "novaclient.client.Client", + "container-infra": "magnumclient.client.Client", "database": "troveclient.client.Client", "identity": "keystoneclient.client.Client", "image": "glanceclient.Client", diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index ba8bf3923..b4e9dea67 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -2,6 +2,7 @@ "auth_type": "password", "baremetal_api_version": "1", "container_api_version": "1", + "container_infra_api_version": "1", "compute_api_version": "2", "database_api_version": "1.0", "disable_vendor_agent": {}, From fcead29c3faa0d7f33e05cd527cbb475542d32ab Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 22 Dec 2016 06:42:25 -0600 Subject: [PATCH 315/365] Update swift constructor to be Session aware python-swiftclient has Session support now! Use it, and remove a pile of customization specific for swift. We still need to put a few parameters in different places, but that's no worse than glance or trove. Change-Id: Ic51aee2bc7b535aa4b6e261fb3deb59bd921f563 --- os_client_config/cloud_config.py | 80 ++++------------- os_client_config/tests/test_cloud_config.py | 96 ++++----------------- 2 files changed, 34 insertions(+), 142 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index f61c49814..ecf60a6e1 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -312,9 +312,6 @@ class CloudConfig(object): if not client_class: client_class = _get_client(service_key) - # Because of course swift is different - if service_key == 'object-store': - return self._get_swift_client(client_class=client_class, **kwargs) interface = self.get_interface(service_key) # trigger exception on lack of service endpoint = self.get_session_endpoint(service_key) @@ -326,12 +323,20 @@ class CloudConfig(object): else: interface_key = 'endpoint_type' - constructor_kwargs = dict( - session=self.get_session(), - service_name=self.get_service_name(service_key), - service_type=self.get_service_type(service_key), - endpoint_override=endpoint_override, - region_name=self.region) + if service_key == 'object-store': + constructor_kwargs = dict( + session=self.get_session(), + os_options=dict( + service_type=self.get_service_type(service_key), + object_storage_url=endpoint_override, + region_name=self.region)) + else: + constructor_kwargs = dict( + session=self.get_session(), + service_name=self.get_service_name(service_key), + service_type=self.get_service_type(service_key), + endpoint_override=endpoint_override, + region_name=self.region) if service_key == 'image': # os-client-config does not depend on glanceclient, but if @@ -350,8 +355,11 @@ class CloudConfig(object): if not endpoint_override: constructor_kwargs['endpoint_override'] = endpoint constructor_kwargs.update(kwargs) - constructor_kwargs[interface_key] = interface - if pass_version_arg: + if service_key == 'object-store': + constructor_kwargs['os_options'][interface_key] = interface + else: + constructor_kwargs[interface_key] = interface + if pass_version_arg and service_key != 'object-store': if not version: version = self.get_api_version(service_key) # Temporary workaround while we wait for python-openstackclient @@ -377,56 +385,6 @@ class CloudConfig(object): return client_class(**constructor_kwargs) - def _get_swift_client(self, client_class, **kwargs): - auth_args = self.get_auth_args() - auth_version = self.get_api_version('identity') - session = self.get_session() - token = session.get_token() - endpoint = self.get_session_endpoint(service_key='object-store') - if not endpoint: - return None - # If we have a username/password, we want to pass them to - # swift - because otherwise it will not re-up tokens appropriately - # However, if we only have non-password auth, then get a token - # and pass it in - swift_kwargs = dict( - auth_version=auth_version, - preauthurl=endpoint, - preauthtoken=token, - os_options=dict( - region_name=self.get_region_name(), - auth_token=token, - object_storage_url=endpoint, - service_type=self.get_service_type('object-store'), - endpoint_type=self.get_interface('object-store'), - - )) - if self.config['api_timeout'] is not None: - swift_kwargs['timeout'] = float(self.config['api_timeout']) - - # create with password - swift_kwargs['user'] = auth_args.get('username') - swift_kwargs['key'] = auth_args.get('password') - swift_kwargs['authurl'] = auth_args.get('auth_url') - os_options = {} - if auth_version == '2.0': - os_options['tenant_name'] = auth_args.get('project_name') - os_options['tenant_id'] = auth_args.get('project_id') - else: - os_options['project_name'] = auth_args.get('project_name') - os_options['project_id'] = auth_args.get('project_id') - - for key in ( - 'user_id', - 'project_domain_id', - 'project_domain_name', - 'user_domain_id', - 'user_domain_name'): - os_options[key] = auth_args.get(key) - swift_kwargs['os_options'].update(os_options) - - return client_class(**swift_kwargs) - def get_cache_expiration_time(self): if self._openstack_config: return self._openstack_config.get_cache_expiration_time() diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index dcbeb3a55..f97f85b67 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -272,26 +272,13 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( - preauthtoken=mock.ANY, - auth_version=u'3', - authurl='http://example.com', - key='testpassword', + session=mock.ANY, os_options={ - 'auth_token': mock.ANY, 'region_name': 'region-al', - 'object_storage_url': 'http://swift.example.com', - 'user_id': None, - 'user_domain_name': None, - 'project_name': 'testproject', - 'project_domain_name': None, - 'project_domain_id': None, - 'project_id': None, 'service_type': 'object-store', + 'object_storage_url': None, 'endpoint_type': 'public', - 'user_domain_id': None - }, - preauthurl='http://swift.example.com', - user='testuser') + }) @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') @@ -311,26 +298,13 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( - preauthtoken=mock.ANY, - auth_version=u'2.0', - authurl='http://example.com', - key='testpassword', + session=mock.ANY, os_options={ - 'auth_token': mock.ANY, 'region_name': 'region-al', - 'object_storage_url': 'http://swift.example.com', - 'user_id': None, - 'user_domain_name': None, - 'tenant_name': 'testproject', - 'project_domain_name': None, - 'project_domain_id': None, - 'tenant_id': None, 'service_type': 'object-store', + 'object_storage_url': None, 'endpoint_type': 'public', - 'user_domain_id': None - }, - preauthurl='http://swift.example.com', - user='testuser') + }) @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') @@ -345,26 +319,13 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( - preauthtoken=mock.ANY, - auth_version=u'2.0', - authurl=None, - key=None, + session=mock.ANY, os_options={ - 'auth_token': mock.ANY, 'region_name': 'region-al', - 'object_storage_url': 'http://example.com/v2', - 'user_id': None, - 'user_domain_name': None, - 'tenant_name': None, - 'project_domain_name': None, - 'project_domain_id': None, - 'tenant_id': None, 'service_type': 'object-store', + 'object_storage_url': None, 'endpoint_type': 'public', - 'user_domain_id': None - }, - preauthurl='http://example.com/v2', - user=None) + }) @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') @@ -380,27 +341,13 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( - preauthtoken=mock.ANY, - auth_version=u'2.0', - authurl=None, - key=None, + session=mock.ANY, os_options={ - 'auth_token': mock.ANY, 'region_name': 'region-al', - 'object_storage_url': 'http://example.com/v2', - 'user_id': None, - 'user_domain_name': None, - 'tenant_name': None, - 'project_domain_name': None, - 'project_domain_id': None, - 'tenant_id': None, 'service_type': 'object-store', + 'object_storage_url': None, 'endpoint_type': 'public', - 'user_domain_id': None - }, - preauthurl='http://example.com/v2', - timeout=9.0, - user=None) + }) @mock.patch.object(cloud_config.CloudConfig, 'get_auth_args') def test_legacy_client_object_store_endpoint( @@ -414,26 +361,13 @@ class TestCloudConfig(base.TestCase): "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_legacy_client('object-store', mock_client) mock_client.assert_called_with( - preauthtoken=mock.ANY, - auth_version=u'2.0', - authurl=None, - key=None, + session=mock.ANY, os_options={ - 'auth_token': mock.ANY, 'region_name': 'region-al', - 'object_storage_url': 'http://example.com/swift', - 'user_id': None, - 'user_domain_name': None, - 'tenant_name': None, - 'project_domain_name': None, - 'project_domain_id': None, - 'tenant_id': None, 'service_type': 'object-store', + 'object_storage_url': 'http://example.com/swift', 'endpoint_type': 'public', - 'user_domain_id': None - }, - preauthurl='http://example.com/swift', - user=None) + }) @mock.patch.object(cloud_config.CloudConfig, 'get_session_endpoint') def test_legacy_client_image(self, mock_get_session_endpoint): From 091fde16a38acf17e0d98db44cedc179464aa2f0 Mon Sep 17 00:00:00 2001 From: avnish Date: Thu, 12 Jan 2017 10:17:37 +0530 Subject: [PATCH 316/365] Use upper-constraints for tox envs Pin tox environments to upper-constraints to avoid conflicts with library releases. Change-Id: I17664e0794de05fb9661050018dff2a07b077826 Closes-Bug: #1628597 --- tools/tox_install.sh | 30 ++++++++++++++++++++++++++++++ tox.ini | 6 +++++- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100755 tools/tox_install.sh diff --git a/tools/tox_install.sh b/tools/tox_install.sh new file mode 100755 index 000000000..43468e450 --- /dev/null +++ b/tools/tox_install.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Client constraint file contains this client version pin that is in conflict +# with installing the client from source. We should remove the version pin in +# the constraints file before applying it for from-source installation. + +CONSTRAINTS_FILE=$1 +shift 1 + +set -e + +# NOTE(tonyb): Place this in the tox enviroment's log dir so it will get +# published to logs.openstack.org for easy debugging. +localfile="$VIRTUAL_ENV/log/upper-constraints.txt" + +if [[ $CONSTRAINTS_FILE != http* ]]; then + CONSTRAINTS_FILE=file://$CONSTRAINTS_FILE +fi +# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep +curl $CONSTRAINTS_FILE --insecure --progress-bar --output $localfile + +pip install -c$localfile openstack-requirements + +# This is the main purpose of the script: Allow local installation of +# the current repo. It is listed in constraints file and thus any +# install will be constrained and we need to unconstrain it. +edit-constraints $localfile -- $CLIENT_NAME + +pip install -c$localfile -U $* +exit $? diff --git a/tox.ini b/tox.ini index cedf47857..08ba56e92 100644 --- a/tox.ini +++ b/tox.ini @@ -5,9 +5,13 @@ skipsdist = True [testenv] usedevelop = True -install_command = pip install -U {opts} {packages} +passenv = ZUUL_CACHE_DIR + REQUIREMENTS_PIP_LOCATION +install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} setenv = VIRTUAL_ENV={envdir} + BRANCH_NAME=master + CLIENT_NAME=os-client-config deps = -r{toxinidir}/test-requirements.txt commands = python setup.py testr --slowest --testr-args='{posargs}' From 53858f340eeb55cf233a56fc6f63bd60f58251b3 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 12 Jan 2017 08:17:22 -0500 Subject: [PATCH 317/365] Remove 3.4 from tox envlist We don't support it. Change-Id: I43c9bc965374cb8fbbe9c40f366aed31d8cf9022 --- setup.cfg | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index f8dcbb096..b87bd6ab3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 [files] packages = diff --git a/tox.ini b/tox.ini index 08ba56e92..bf0fd8bf9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py34,py35,py27,pypy,pep8 +envlist = py35,py27,pypy,pep8 skipsdist = True [testenv] From bedc9c57c2f28f30bdc200f7c6edf653db4cde90 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 19 Oct 2016 16:17:13 -0500 Subject: [PATCH 318/365] Add OpenTelekomCloud to the vendors Change-Id: I82fad53ad2078f58ba14e16a7199b7b730a37457 --- doc/source/vendor-support.rst | 15 +++++++++++++++ os_client_config/vendors/otc.json | 13 +++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 os_client_config/vendors/otc.json diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index b301d8017..ff0669102 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -134,6 +134,21 @@ RegionOne Ashburn, VA * Public IPv4 is provided via NAT with Neutron Floating IP * IPv6 is provided to every server +otc +--- + +https://iam.%(region_name)s.otc.t-systems.com/v3 + +============== ================ +Region Name Location +============== ================ +eu-de Germany +============== ================ + +* Identity API Version is 3 +* Images must be in `vhd` format +* Public IPv4 is provided via NAT with Neutron Floating IP + elastx ------ diff --git a/os_client_config/vendors/otc.json b/os_client_config/vendors/otc.json new file mode 100644 index 000000000..b0c1b116f --- /dev/null +++ b/os_client_config/vendors/otc.json @@ -0,0 +1,13 @@ +{ + "name": "otc", + "profile": { + "auth": { + "auth_url": "https://iam.%(region_name)s.otc.t-systems.com/v3" + }, + "regions": [ + "eu-de" + ], + "identity_api_version": "3", + "image_format": "vhd" + } +} From cc2b337dcd55d8d2edd53b6000f1dd128636b529 Mon Sep 17 00:00:00 2001 From: Andy Botting Date: Wed, 18 Jan 2017 15:54:00 +1100 Subject: [PATCH 319/365] Add support for Murano Add Murano support with the service name 'application-catalog' Change-Id: I42794993b8f6208d40786e83ec80ee64a0879415 --- os_client_config/constructors.json | 1 + os_client_config/defaults.json | 1 + 2 files changed, 2 insertions(+) diff --git a/os_client_config/constructors.json b/os_client_config/constructors.json index a78be6bcb..7ad420e61 100644 --- a/os_client_config/constructors.json +++ b/os_client_config/constructors.json @@ -1,4 +1,5 @@ { + "application-catalog": "muranoclient.client.Client", "compute": "novaclient.client.Client", "container-infra": "magnumclient.client.Client", "database": "troveclient.client.Client", diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index b4e9dea67..65f896151 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -1,4 +1,5 @@ { + "application_catalog_api_version": "1", "auth_type": "password", "baremetal_api_version": "1", "container_api_version": "1", From 3c47e251c978cf9c5e393f032c0c70d67c1dcb00 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 18 Jan 2017 14:25:53 -0600 Subject: [PATCH 320/365] Revert "Fix interface_key for identity clients" This patch breaks stable/newton devstack-gate of shade. This reverts commit fa4e1bd21db4bee2a0ee779067cdd659e647d7fc. Change-Id: I31a7831693f567a0717a9b41c242453fb937d6d7 --- os_client_config/cloud_config.py | 2 +- os_client_config/tests/test_cloud_config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index ecf60a6e1..a7fc0582f 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -318,7 +318,7 @@ class CloudConfig(object): endpoint_override = self.get_endpoint(service_key) if not interface_key: - if service_key in ('image', 'key-manager', 'identity'): + if service_key in ('image', 'key-manager'): interface_key = 'interface' else: interface_key = 'endpoint_type' diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index f97f85b67..6f960e6e1 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -524,7 +524,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='2.0', endpoint='http://example.com/v2', - interface='admin', + endpoint_type='admin', endpoint_override=None, region_name='region-al', service_type='identity', @@ -544,7 +544,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='3', endpoint='http://example.com', - interface='admin', + endpoint_type='admin', endpoint_override=None, region_name='region-al', service_type='identity', From 08b7ce9331a32731533cc5158d3fa3d3bf03a0d3 Mon Sep 17 00:00:00 2001 From: Andy Botting Date: Thu, 19 Jan 2017 15:30:25 +1100 Subject: [PATCH 321/365] Fix typo for baremetal_service_type Fix a copy-and-paste error for the baremetal service. Change-Id: Ifbef9d0ad01c57bd98f06f7f10f9d632753d8221 --- os_client_config/vendor-schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/vendor-schema.json b/os_client_config/vendor-schema.json index 6a6f4561e..692b79c51 100644 --- a/os_client_config/vendor-schema.json +++ b/os_client_config/vendor-schema.json @@ -157,7 +157,7 @@ "description": "Object Storage API Service Type", "type": "string" }, - "baremetal_api_version": { + "baremetal_service_type": { "name": "Baremetal API Service Type", "description": "Baremetal API Service Type", "type": "string" From 02116c41ef7cbf05698cfd01d28ce276d3f4f23d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 31 Jan 2017 16:03:05 -0500 Subject: [PATCH 322/365] fix location of team tags in README Remove the extraneous title markup and move the team tag include instructions below the main project title in the readme so it renders more nicely. Change-Id: Icd384c81a455a3e1a86abd1f2ef84e775e06c307 Signed-off-by: Doug Hellmann --- README.rst | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 9e460be85..67aa91a34 100644 --- a/README.rst +++ b/README.rst @@ -1,16 +1,10 @@ -======================== -Team and repository tags -======================== - -.. image:: http://governance.openstack.org/badges/os-client-config.svg - :target: http://governance.openstack.org/reference/tags/index.html - -.. Change things from this point on - ================ os-client-config ================ +.. image:: http://governance.openstack.org/badges/os-client-config.svg + :target: http://governance.openstack.org/reference/tags/index.html + `os-client-config` is a library for collecting client configuration for using an OpenStack cloud in a consistent and comprehensive manner. It will find cloud config for as few as 1 cloud and as many as you want to From 707adab1bc8726a7bf9c910e5bd2650bfe420ad1 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Tue, 31 Jan 2017 21:04:24 +0000 Subject: [PATCH 323/365] Update reno for stable/ocata Change-Id: Iace25f1919632b5de8d6bf81add7ce0416a874ef --- releasenotes/source/index.rst | 1 + releasenotes/source/ocata.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/ocata.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index f708ec8d3..22609515d 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -9,6 +9,7 @@ Contents :maxdepth: 2 unreleased + ocata newton mitaka diff --git a/releasenotes/source/ocata.rst b/releasenotes/source/ocata.rst new file mode 100644 index 000000000..ebe62f42e --- /dev/null +++ b/releasenotes/source/ocata.rst @@ -0,0 +1,6 @@ +=================================== + Ocata Series Release Notes +=================================== + +.. release-notes:: + :branch: origin/stable/ocata From 50efb434d874ca74e6c89424c43389408ab9b584 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 8 Feb 2017 08:25:21 -0600 Subject: [PATCH 324/365] Add support for indicating required floating IPs Some clouds require that users add a floating IP to a server if the user wants that server to be able to talk to things that are not on the cloud. Some clouds do not require this and instead give servers a directly attached IP. The only way a user can know is to boot a server, then ask neutron for the port associated with that server, then find the network the port came from and then try to infer whether or not that network has the ability to route packets northbound. Of course, networks don't actually communicate that quality directly, (router:external doesn't mean a network routes externally, it means the network can have a router attached to it to provide floating ips) so it's still hit and miss. Where we can, save the user the stress and strain of not knowing how their cloud wants them to get an externally routable IP. Change-Id: I1baf804ce28bc1997b2347c4648c5cc56c750ead --- doc/source/vendor-support.rst | 1 + os_client_config/cloud_config.py | 13 ++++++++++++- os_client_config/vendor-schema.json | 6 ++++++ os_client_config/vendors/auro.json | 3 ++- os_client_config/vendors/citycloud.json | 1 + os_client_config/vendors/rackspace.json | 1 + os_client_config/vendors/vexxhost.json | 3 ++- 7 files changed, 25 insertions(+), 3 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index ff0669102..577093c57 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -21,6 +21,7 @@ These are the default behaviors unless a cloud is configured differently. * Images are uploaded using PUT interface * Public IPv4 is directly routable via DHCP from Neutron * IPv6 is not provided +* Floating IPs are not required * Floating IPs are provided by Neutron * Security groups are provided by Neutron * Vendor specific agents are not used diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index a7fc0582f..f52756fd8 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -244,7 +244,6 @@ class CloudConfig(object): :param service_key: Generic key for service, such as 'compute' or 'network' - :returns: Endpoint for the service, or None if not found """ override_endpoint = self.get_endpoint(service_key) @@ -420,6 +419,18 @@ class CloudConfig(object): return default return float(expiration[resource]) + def requires_floating_ip(self): + """Return whether or not this cloud requires floating ips. + + + :returns: True of False if know, None if discovery is needed. + If requires_floating_ip is not configured but the cloud is + known to not provide floating ips, will return False. + """ + if self.config['floating_ip_source'] == "None": + return False + return self.config['requires_floating_ip'] + def get_external_networks(self): """Get list of network names for external networks.""" return [ diff --git a/os_client_config/vendor-schema.json b/os_client_config/vendor-schema.json index 692b79c51..8193a19ba 100644 --- a/os_client_config/vendor-schema.json +++ b/os_client_config/vendor-schema.json @@ -60,6 +60,12 @@ "description": "Optional message with information related to status", "type": "string" }, + "requires_floating_ip": { + "name": "Requires Floating IP", + "description": "Whether the cloud requires a floating IP to route traffic off of the cloud", + "default": null, + "type": ["boolean", "null"] + }, "secgroup_source": { "name": "Security Group Source", "description": "Which service provides security groups", diff --git a/os_client_config/vendors/auro.json b/os_client_config/vendors/auro.json index a9e709bea..410a8e19c 100644 --- a/os_client_config/vendors/auro.json +++ b/os_client_config/vendors/auro.json @@ -5,6 +5,7 @@ "auth_url": "https://api.van1.auro.io:5000/v2.0" }, "identity_api_version": "2", - "region_name": "van1" + "region_name": "van1", + "requires_floating_ip": true } } diff --git a/os_client_config/vendors/citycloud.json b/os_client_config/vendors/citycloud.json index 097ddfdb3..c9ac335c8 100644 --- a/os_client_config/vendors/citycloud.json +++ b/os_client_config/vendors/citycloud.json @@ -12,6 +12,7 @@ "Sto2", "Kna1" ], + "requires_floating_ip": true, "volume_api_version": "1", "identity_api_version": "3" } diff --git a/os_client_config/vendors/rackspace.json b/os_client_config/vendors/rackspace.json index 3fbbacd90..6a4590f67 100644 --- a/os_client_config/vendors/rackspace.json +++ b/os_client_config/vendors/rackspace.json @@ -18,6 +18,7 @@ "image_format": "vhd", "floating_ip_source": "None", "secgroup_source": "None", + "requires_floating_ip": false, "volume_api_version": "1", "disable_vendor_agent": { "vm_mode": "hvm", diff --git a/os_client_config/vendors/vexxhost.json b/os_client_config/vendors/vexxhost.json index aa2cedc68..2227fff4f 100644 --- a/os_client_config/vendors/vexxhost.json +++ b/os_client_config/vendors/vexxhost.json @@ -9,6 +9,7 @@ ], "dns_api_version": "1", "identity_api_version": "3", - "floating_ip_source": "None" + "floating_ip_source": "None", + "requires_floating_ip": false } } From cbb38f38bceee7977952651bd0466902c087de8d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 9 Feb 2017 13:13:58 -0600 Subject: [PATCH 325/365] Add helper scripts to print version discovery info These are simple scripts I made to investigate things. Each show the version discovery info for all of the clouds in a clouds.yaml. Change-Id: I742a59c737c53c05851015b9734c7aa85a5466ca --- tools/keystone_version.py | 89 +++++++++++++++++++++++++++++++++++++++ tools/nova_version.py | 56 ++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 tools/keystone_version.py create mode 100644 tools/nova_version.py diff --git a/tools/keystone_version.py b/tools/keystone_version.py new file mode 100644 index 000000000..a81bdabf2 --- /dev/null +++ b/tools/keystone_version.py @@ -0,0 +1,89 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os_client_config +import pprint +import sys +import urlparse + + +def print_versions(r): + if 'version' in r: + for version in r['version']: + print_version(version) + if 'values' in r: + for version in r['values']: + print_version(version) + if isinstance(r, list): + for version in r: + print_version(version) + + +def print_version(version): + if version['status'] in ('CURRENT', 'stable'): + print( + "\tVersion ID: {id} updated {updated}".format( + id=version.get('id'), + updated=version.get('updated'))) + + +verbose = '-v' in sys.argv +ran = [] +for cloud in os_client_config.OpenStackConfig().get_all_clouds(): + if cloud.name in ran: + continue + ran.append(cloud.name) + # We don't actually need a compute client - but we'll be getting full urls + # anyway. Without this SSL cert info becomes wrong. + c = cloud.get_session_client('compute') + endpoint = cloud.config['auth']['auth_url'] + try: + print(endpoint) + r = c.get(endpoint).json() + if verbose: + pprint.pprint(r) + except Exception as e: + print("Error with {cloud}: {e}".format(cloud=cloud.name, e=str(e))) + continue + if 'version' in r: + print_version(r['version']) + url = urlparse.urlparse(endpoint) + parts = url.path.split(':') + if len(parts) == 2: + path, port = parts + else: + path = url.path + port = None + stripped = path.rsplit('/', 2)[0] + if port: + stripped = '{stripped}:{port}'.format(stripped=stripped, port=port) + endpoint = urlparse.urlunsplit( + (url.scheme, url.netloc, stripped, url.params, url.query)) + print(" also {endpoint}".format(endpoint=endpoint)) + try: + r = c.get(endpoint).json() + if verbose: + pprint.pprint(r) + except Exception: + print("\tUnauthorized") + continue + if 'version' in r: + print_version(r) + elif 'versions' in r: + print_versions(r['versions']) + else: + print("\n\nUNKNOWN\n\n{r}".format(r=r)) + else: + print_versions(r['versions']) diff --git a/tools/nova_version.py b/tools/nova_version.py new file mode 100644 index 000000000..20603db41 --- /dev/null +++ b/tools/nova_version.py @@ -0,0 +1,56 @@ +# Copyright (c) 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os_client_config + +ran = [] +for cloud in os_client_config.OpenStackConfig().get_all_clouds(): + if cloud.name in ran: + continue + ran.append(cloud.name) + c = cloud.get_session_client('compute') + try: + raw_endpoint = c.get_endpoint() + have_current = False + endpoint = raw_endpoint.rsplit('/', 2)[0] + print(endpoint) + r = c.get(endpoint).json() + except Exception: + print("Error with %s" % cloud.name) + continue + for version in r['versions']: + if version['status'] == 'CURRENT': + have_current = True + print( + "\tVersion ID: {id} updated {updated}".format( + id=version.get('id'), + updated=version.get('updated'))) + print( + "\tVersion Max: {max}".format(max=version.get('version'))) + print( + "\tVersion Min: {min}".format(min=version.get('min_version'))) + if not have_current: + for version in r['versions']: + if version['status'] == 'SUPPORTED': + have_current = True + print( + "\tVersion ID: {id} updated {updated}".format( + id=version.get('id'), + updated=version.get('updated'))) + print( + "\tVersion Max: {max}".format(max=version.get('version'))) + print( + "\tVersion Min: {min}".format( + min=version.get('min_version'))) From 0e039e67c2e3496628141b7b48aff2ff541096c6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 15 Feb 2017 09:31:12 -0600 Subject: [PATCH 326/365] Add support for overriding mistral service type The mistral team copied the heinous pervsion that the cinder team propagated upon the world and appended a version to their service_type. That's ok - there is nice copy-pastable code here we can use to prevent users from feeling the pain. Change-Id: Icf280f932014e4d9abeab3e944aece125988562e --- os_client_config/cloud_config.py | 4 ++++ os_client_config/defaults.json | 3 ++- os_client_config/tests/test_cloud_config.py | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index f52756fd8..5c4c03e52 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -161,11 +161,15 @@ class CloudConfig(object): # Of course, if the user requests a volumev2, that structure should # still work. # What's even more amazing is that they did it AGAIN with cinder v3 + # And then I learned that mistral copied it. if service_type == 'volume': if self.get_api_version(service_type).startswith('2'): service_type = 'volumev2' elif self.get_api_version(service_type).startswith('3'): service_type = 'volumev3' + elif service_type == 'workflow': + if self.get_api_version(service_type).startswith('2'): + service_type = 'workflowv2' return self.config.get(key, service_type) def get_service_name(self, service_type): diff --git a/os_client_config/defaults.json b/os_client_config/defaults.json index 65f896151..2a195c426 100644 --- a/os_client_config/defaults.json +++ b/os_client_config/defaults.json @@ -22,5 +22,6 @@ "orchestration_api_version": "1", "secgroup_source": "neutron", "status": "active", - "volume_api_version": "2" + "volume_api_version": "2", + "workflow_api_version": "2" } diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 6f960e6e1..ce724cb56 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -163,6 +163,11 @@ class TestCloudConfig(base.TestCase): cc.config['volume_api_version'] = '3' self.assertEqual('volumev3', cc.get_service_type('volume')) + def test_workflow_override_v2(self): + cc = cloud_config.CloudConfig("test1", "region-al", fake_services_dict) + cc.config['workflow_api_version'] = '2' + self.assertEqual('workflowv2', cc.get_service_type('workflow')) + def test_get_session_no_auth(self): config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) From e6755872ada4978f585bdf15edf623dbcf72c4ee Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 15 Feb 2017 10:30:12 -0600 Subject: [PATCH 327/365] Remove the keystoneclient auth fallback OSC doesn't use this codepath anyway (to my knowledge) and it masks errors in exceptionally strange ways. Change-Id: I15ec5aacb037813a98ac9ea8e9504a5d1cc90837 --- os_client_config/config.py | 71 ++------------------------- os_client_config/tests/test_config.py | 12 ++--- os_client_config/tests/test_init.py | 6 ++- test-requirements.txt | 1 - 4 files changed, 13 insertions(+), 77 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 9b4a709fd..de8dd3806 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -861,59 +861,6 @@ class OpenStackConfig(object): config['auth_type'] = 'admin_token' return loading.get_plugin_loader(config['auth_type']) - def _validate_auth_ksc(self, config, cloud): - try: - import keystoneclient.auth as ksc_auth - except ImportError: - return config - - # May throw a keystoneclient.exceptions.NoMatchingPlugin - plugin_options = ksc_auth.get_plugin_class( - config['auth_type']).get_options() - - for p_opt in plugin_options: - # if it's in config.auth, win, kill it from config dict - # if it's in config and not in config.auth, move it - # deprecated loses to current - # provided beats default, deprecated or not - winning_value = self._find_winning_auth_value( - p_opt, - config['auth'], - ) - if not winning_value: - winning_value = self._find_winning_auth_value( - p_opt, - config, - ) - - # if the plugin tells us that this value is required - # then error if it's doesn't exist now - if not winning_value and p_opt.required: - raise exceptions.OpenStackConfigException( - 'Unable to find auth information for cloud' - ' {cloud} in config files {files}' - ' or environment variables. Missing value {auth_key}' - ' required for auth plugin {plugin}'.format( - cloud=cloud, files=','.join(self._config_files), - auth_key=p_opt.name, plugin=config.get('auth_type'))) - - # Clean up after ourselves - for opt in [p_opt.name] + [o.name for o in p_opt.deprecated_opts]: - opt = opt.replace('-', '_') - config.pop(opt, None) - config['auth'].pop(opt, None) - - if winning_value: - # Prefer the plugin configuration dest value if the value's key - # is marked as depreciated. - if p_opt.dest is None: - config['auth'][p_opt.name.replace('-', '_')] = ( - winning_value) - else: - config['auth'][p_opt.dest] = winning_value - - return config - def _validate_auth(self, config, loader): # May throw a keystoneauth1.exceptions.NoMatchingPlugin @@ -1107,21 +1054,9 @@ class OpenStackConfig(object): config = self.auth_config_hook(config) if validate: - try: - loader = self._get_auth_loader(config) - config = self._validate_auth(config, loader) - auth_plugin = loader.load_from_options(**config['auth']) - except Exception as e: - # We WANT the ksa exception normally - # but OSC can't handle it right now, so we try deferring - # to ksc. If that ALSO fails, it means there is likely - # a deeper issue, so we assume the ksa error was correct - self.log.debug("Deferring keystone exception: {e}".format(e=e)) - auth_plugin = None - try: - config = self._validate_auth_ksc(config, cloud) - except Exception: - raise e + loader = self._get_auth_loader(config) + config = self._validate_auth(config, loader) + auth_plugin = loader.load_from_options(**config['auth']) else: auth_plugin = None diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index ad3685ab1..aa5935acf 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -226,7 +226,7 @@ class TestConfig(base.TestCase): c = config.OpenStackConfig(config_files=['nonexistent'], vendor_files=['nonexistent'], secure_files=[self.secure_yaml]) - cc = c.get_one_cloud(cloud='_test_cloud_no_vendor') + cc = c.get_one_cloud(cloud='_test_cloud_no_vendor', validate=False) self.assertEqual('testpass', cc.auth['password']) def test_get_cloud_names(self): @@ -384,7 +384,7 @@ class TestConfigArgparse(base.TestCase): vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud( - cloud='_test_cloud_regions', argparse=self.options) + cloud='_test_cloud_regions', argparse=self.options, validate=False) self.assertEqual(cc.region_name, 'region2') self.assertEqual(cc.snack_type, 'cookie') @@ -481,7 +481,7 @@ class TestConfigArgparse(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud(argparse=self.options) + cc = c.get_one_cloud(argparse=self.options, validate=False) self.assertIsNone(cc.cloud) self.assertEqual(cc.region_name, 'region2') self.assertEqual(cc.snack_type, 'cookie') @@ -490,7 +490,7 @@ class TestConfigArgparse(base.TestCase): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) - cc = c.get_one_cloud(**self.args) + cc = c.get_one_cloud(validate=False, **self.args) self.assertIsNone(cc.cloud) self.assertEqual(cc.region_name, 'region2') self.assertEqual(cc.snack_type, 'cookie') @@ -622,7 +622,7 @@ class TestConfigArgparse(base.TestCase): vendor_files=[self.vendor_yaml]) cc = c.get_one_cloud( - cloud='envvars', argparse=self.options) + cloud='envvars', argparse=self.options, validate=False) self.assertEqual(cc.auth['project_name'], 'project') def test_argparse_default_no_token(self): @@ -650,7 +650,7 @@ class TestConfigArgparse(base.TestCase): opts, _remain = parser.parse_known_args( ['--os-auth-token', 'very-bad-things', '--os-auth-type', 'token']) - cc = c.get_one_cloud(argparse=opts) + cc = c.get_one_cloud(argparse=opts, validate=False) self.assertEqual(cc.config['auth_type'], 'token') self.assertEqual(cc.config['auth']['token'], 'very-bad-things') diff --git a/os_client_config/tests/test_init.py b/os_client_config/tests/test_init.py index 15d57f717..5b4fab998 100644 --- a/os_client_config/tests/test_init.py +++ b/os_client_config/tests/test_init.py @@ -18,7 +18,8 @@ from os_client_config.tests import base class TestInit(base.TestCase): def test_get_config_without_arg_parser(self): - cloud_config = os_client_config.get_config(options=None) + cloud_config = os_client_config.get_config( + options=None, validate=False) self.assertIsInstance( cloud_config, os_client_config.cloud_config.CloudConfig @@ -26,7 +27,8 @@ class TestInit(base.TestCase): def test_get_config_with_arg_parser(self): cloud_config = os_client_config.get_config( - options=argparse.ArgumentParser()) + options=argparse.ArgumentParser(), + validate=False) self.assertIsInstance( cloud_config, os_client_config.cloud_config.CloudConfig diff --git a/test-requirements.txt b/test-requirements.txt index f9908d6e0..dd58260f4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -11,7 +11,6 @@ fixtures>=0.3.14 jsonschema>=2.0.0,<3.0.0,!=2.5.0 mock>=1.2 python-glanceclient>=0.18.0 -python-keystoneclient>=1.1.0 python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 oslosphinx>=2.5.0,<2.6.0 # Apache-2.0 From 81e04fcec6cb333a7d44124c045358c4904cf765 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 27 Feb 2017 09:34:24 -0600 Subject: [PATCH 328/365] Add ability to skip yaml loading Added a flag, 'load_yaml_config' that defaults to True. If set to false, no clouds.yaml files will be loaded. This is beneficial if os-client-config wants to be used inside of a service where end-user clouds.yaml files would make things more confusing. Change-Id: Idbc82bb931e9edf1bbcc575237c0e202e219c218 --- os_client_config/__init__.py | 3 +- os_client_config/config.py | 14 +++++--- os_client_config/tests/test_config.py | 32 +++++++++++++++++++ .../notes/load-yaml-3177efca78e5c67a.yaml | 7 ++++ 4 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/load-yaml-3177efca78e5c67a.yaml diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index e8d7fc0f8..a36a13061 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -24,7 +24,8 @@ __version__ = pbr.version.VersionInfo('os_client_config').version_string() def get_config(service_key=None, options=None, **kwargs): - config = OpenStackConfig() + load_yaml_config = kwargs.pop('load_yaml_config', True) + config = OpenStackConfig(load_yaml_config=load_yaml_config) if options: config.register_argparse_arguments(options, sys.argv, service_key) parsed_options = options.parse_known_args(sys.argv) diff --git a/os_client_config/config.py b/os_client_config/config.py index 9b4a709fd..0e241dfee 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -173,13 +173,19 @@ class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, override_defaults=None, force_ipv4=None, envvar_prefix=None, secure_files=None, - pw_func=None, session_constructor=None): + pw_func=None, session_constructor=None, + load_yaml_config=True): self.log = _log.setup_logging(__name__) self._session_constructor = session_constructor - self._config_files = config_files or CONFIG_FILES - self._secure_files = secure_files or SECURE_FILES - self._vendor_files = vendor_files or VENDOR_FILES + if load_yaml_config: + self._config_files = config_files or CONFIG_FILES + self._secure_files = secure_files or SECURE_FILES + self._vendor_files = vendor_files or VENDOR_FILES + else: + self._config_files = [] + self._secure_files = [] + self._vendor_files = [] config_file_override = os.environ.pop('OS_CLIENT_CONFIG_FILE', None) if config_file_override: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index ad3685ab1..195488d92 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -16,6 +16,7 @@ import argparse import copy import os +import extras import fixtures import testtools import yaml @@ -577,6 +578,37 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(cc.region_name, 'region2') self.assertEqual('my-network', cc.config['external_network']) + def test_get_one_cloud_no_yaml_no_cloud(self): + c = config.OpenStackConfig(load_yaml_config=False) + + self.assertRaises( + exceptions.OpenStackConfigException, + c.get_one_cloud, + cloud='_test_cloud_regions', region_name='region2', argparse=None) + + def test_get_one_cloud_no_yaml(self): + c = config.OpenStackConfig(load_yaml_config=False) + + cc = c.get_one_cloud( + region_name='region2', argparse=None, + **base.USER_CONF['clouds']['_test_cloud_regions']) + # Not using assert_cloud_details because of cache settings which + # are not present without the file + self.assertIsInstance(cc, cloud_config.CloudConfig) + self.assertTrue(extras.safe_hasattr(cc, 'auth')) + self.assertIsInstance(cc.auth, dict) + self.assertIsNone(cc.cloud) + self.assertIn('username', cc.auth) + self.assertEqual('testuser', cc.auth['username']) + self.assertEqual('testpass', cc.auth['password']) + self.assertFalse(cc.config['image_api_use_tasks']) + self.assertTrue('project_name' in cc.auth or 'project_id' in cc.auth) + if 'project_name' in cc.auth: + self.assertEqual('testproject', cc.auth['project_name']) + elif 'project_id' in cc.auth: + self.assertEqual('testproject', cc.auth['project_id']) + self.assertEqual(cc.region_name, 'region2') + def test_fix_env_args(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml]) diff --git a/releasenotes/notes/load-yaml-3177efca78e5c67a.yaml b/releasenotes/notes/load-yaml-3177efca78e5c67a.yaml new file mode 100644 index 000000000..2438f83a4 --- /dev/null +++ b/releasenotes/notes/load-yaml-3177efca78e5c67a.yaml @@ -0,0 +1,7 @@ +--- +features: + - Added a flag, 'load_yaml_config' that defaults to True. + If set to false, no clouds.yaml files will be loaded. This + is beneficial if os-client-config wants to be used inside of + a service where end-user clouds.yaml files would make things + more confusing. From d321a14ecbe79c888e891424250c1b5bdfd2ea65 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 28 Feb 2017 11:22:30 -0600 Subject: [PATCH 329/365] Pass ironic microversion through from api_version If someone sets baremetal_api_version to 1.29 right now, we don't really do anything with that information. Pass it through to the constructor for ironicclient in get_legacy_client(). Change-Id: I470fbb8852eac7d5cb35aef549ac591d63f3636f --- os_client_config/cloud_config.py | 6 ++++++ .../notes/ironic-microversion-ba5b0f36f11196a6.yaml | 4 ++++ 2 files changed, 10 insertions(+) create mode 100644 releasenotes/notes/ironic-microversion-ba5b0f36f11196a6.yaml diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index f52756fd8..3521920d4 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -372,6 +372,12 @@ class CloudConfig(object): constructor_kwargs['endpoint'] = endpoint if service_key == 'network': constructor_kwargs['api_version'] = version + elif service_key == 'baremetal': + if version != '1': + # Set Ironic Microversion + constructor_kwargs['os_ironic_api_version'] = version + # Version arg is the major version, not the full microstring + constructor_kwargs['version'] = version[0] else: constructor_kwargs['version'] = version if service_key == 'database': diff --git a/releasenotes/notes/ironic-microversion-ba5b0f36f11196a6.yaml b/releasenotes/notes/ironic-microversion-ba5b0f36f11196a6.yaml new file mode 100644 index 000000000..62e36277d --- /dev/null +++ b/releasenotes/notes/ironic-microversion-ba5b0f36f11196a6.yaml @@ -0,0 +1,4 @@ +--- +features: + - Add support for passing Ironic microversion to the ironicclient + constructor in get_legacy_client. From 40c416cbad77d74d7d8fb964193b309d77ecb628 Mon Sep 17 00:00:00 2001 From: ricolin Date: Thu, 2 Mar 2017 22:26:45 +0800 Subject: [PATCH 330/365] [Fix gate]Update test requirement Since pbr already landed and the old version of hacking seems not work very well with pbr>=2, we should update it to match global requirement. Partial-Bug: #1668848 Change-Id: I09ae994782889aae05250a8e5bf9f5b630b2d502 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index f9908d6e0..6208fb511 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=0.10.2,<0.11 # Apache-2.0 +hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0 coverage>=3.6 docutils>=0.11,!=0.13.1 # OSI-Approved Open Source, Public Domain From 0a956c1d281d8ffe984fac0bd5588fad5d00a00d Mon Sep 17 00:00:00 2001 From: xhzhf Date: Tue, 21 Feb 2017 17:44:56 +0800 Subject: [PATCH 331/365] modify test-requirement according to requirements project the OpenStack requirements project has modify version requirement of docutils/oslosphinx. The link is below https://review.openstack.org/#/c/418772/ So modify test-requirement like other project Closes-Bug: #1666149 Change-Id: I145ba596926cac0efab75fb4a1548eea338a2d5a --- test-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index f9908d6e0..10abdfe3d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,7 +5,7 @@ hacking>=0.10.2,<0.11 # Apache-2.0 coverage>=3.6 -docutils>=0.11,!=0.13.1 # OSI-Approved Open Source, Public Domain +docutils>=0.11 # OSI-Approved Open Source, Public Domain extras fixtures>=0.3.14 jsonschema>=2.0.0,<3.0.0,!=2.5.0 @@ -13,8 +13,8 @@ mock>=1.2 python-glanceclient>=0.18.0 python-keystoneclient>=1.1.0 python-subunit>=0.0.18 -sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 -oslosphinx>=2.5.0,<2.6.0 # Apache-2.0 +sphinx>=1.5.1 # BSD +oslosphinx>=4.7.0 # Apache-2.0 oslotest>=1.5.1,<1.6.0 # Apache-2.0 reno>=0.1.1 # Apache2 testrepository>=0.0.18 From b2f7ceadb1a99bd0f5fb17b9298c6f962414aa9f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 28 Feb 2017 09:07:27 -0600 Subject: [PATCH 332/365] Add support for bailing on invalid service versions At least for cinder for now, allow a consumer of get_legacy_client to express the minimum version they find acceptable. This will use cinder_client logic to figure out the version from the url. As a follow on, expand this to all of the clients and make it support microversions for the clients that support microversions. (Right now it's just going to be major versions, so min_version=1 will throw an exception if the cinder service returns a v1 endpoint. Also, because we override the volume/volumev2/volumev3 service type stuff, we need to do extra special logic in get_session_endpoint to try all three in the case where do not have a configured api_version. Change-Id: I7b6b3588fec9a6be892cf20d344667f0b9a62f0a --- os_client_config/cloud_config.py | 87 +++++++++++++++---- os_client_config/exceptions.py | 8 ++ ...n-max-legacy-version-301242466ddefa93.yaml | 15 ++++ 3 files changed, 93 insertions(+), 17 deletions(-) create mode 100644 releasenotes/notes/min-max-legacy-version-301242466ddefa93.yaml diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 3521920d4..22b7d4aab 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -13,6 +13,7 @@ # under the License. import importlib +import math import warnings from keystoneauth1 import adapter @@ -234,7 +235,19 @@ class CloudConfig(object): interface=self.get_interface(service_key), region_name=self.region) - def get_session_endpoint(self, service_key): + def _get_highest_endpoint(self, service_types, kwargs): + session = self.get_session() + for service_type in service_types: + kwargs['service_type'] = service_type + try: + # Return the highest version we find that matches + # the request + return session.get_endpoint(**kwargs) + except keystoneauth1.exceptions.catalog.EndpointNotFound: + pass + + def get_session_endpoint( + self, service_key, min_version=None, max_version=None): """Return the endpoint from config or the catalog. If a configuration lists an explicit endpoint for a service, @@ -250,27 +263,51 @@ class CloudConfig(object): if override_endpoint: return override_endpoint # keystone is a special case in keystone, because what? - session = self.get_session() + endpoint = None + kwargs = { + 'service_name': self.get_service_name(service_key), + 'region_name': self.region + } if service_key == 'identity': - endpoint = session.get_endpoint(interface=plugin.AUTH_INTERFACE) + # setting interface in kwargs dict even though we don't use kwargs + # dict here just for ease of warning text later + kwargs['interface'] = plugin.AUTH_INTERFACE + session = self.get_session() + endpoint = session.get_endpoint(interface=kwargs['interface']) else: - args = { - 'service_type': self.get_service_type(service_key), - 'service_name': self.get_service_name(service_key), - 'interface': self.get_interface(service_key), - 'region_name': self.region - } - try: - endpoint = session.get_endpoint(**args) - except keystoneauth1.exceptions.catalog.EndpointNotFound: - self.log.warning("Keystone catalog entry not found (%s)", - args) - endpoint = None + kwargs['interface'] = self.get_interface(service_key) + if service_key == 'volume' and not self.get_api_version('volume'): + # If we don't have a configured cinder version, we can't know + # to request a different service_type + min_version = float(min_version or 1) + max_version = float(max_version or 3) + min_major = math.trunc(float(min_version)) + max_major = math.trunc(float(max_version)) + versions = range(int(max_major) + 1, int(min_major), -1) + service_types = [] + for version in versions: + if version == 1: + service_types.append('volume') + else: + service_types.append('volumev{v}'.format(v=version)) + else: + service_types = [self.get_service_type(service_key)] + endpoint = self._get_highest_endpoint(service_types, kwargs) + if not endpoint: + self.log.warning( + "Keystone catalog entry not found (" + "service_type=%s,service_name=%s" + "interface=%s,region_name=%s)", + service_key, + kwargs['service_name'], + kwargs['interface'], + kwargs['region_name']) return endpoint def get_legacy_client( self, service_key, client_class=None, interface_key=None, - pass_version_arg=True, version=None, **kwargs): + pass_version_arg=True, version=None, min_version=None, + max_version=None, **kwargs): """Return a legacy OpenStack client object for the given config. Most of the OpenStack python-*client libraries have the same @@ -304,6 +341,8 @@ class CloudConfig(object): that case. :param version: (optional) Version string to override the configured version string. + :param min_version: (options) Minimum version acceptable. + :param max_version: (options) Maximum version acceptable. :param kwargs: (optional) keyword args are passed through to the Client constructor, so this is in case anything additional needs to be passed in. @@ -313,7 +352,8 @@ class CloudConfig(object): interface = self.get_interface(service_key) # trigger exception on lack of service - endpoint = self.get_session_endpoint(service_key) + endpoint = self.get_session_endpoint( + service_key, min_version=min_version, max_version=max_version) endpoint_override = self.get_endpoint(service_key) if not interface_key: @@ -361,6 +401,9 @@ class CloudConfig(object): if pass_version_arg and service_key != 'object-store': if not version: version = self.get_api_version(service_key) + if not version and service_key == 'volume': + from cinderclient import client as cinder_client + version = cinder_client.get_volume_api_from_url(endpoint) # Temporary workaround while we wait for python-openstackclient # to be able to handle 2.0 which is what neutronclient expects if service_key == 'network' and version == '2': @@ -380,6 +423,16 @@ class CloudConfig(object): constructor_kwargs['version'] = version[0] else: constructor_kwargs['version'] = version + if min_version and min_version > float(version): + raise exceptions.OpenStackConfigVersionException( + "Minimum version {min_version} requested but {version}" + " found".format(min_version=min_version, version=version), + version=version) + if max_version and max_version < float(version): + raise exceptions.OpenStackConfigVersionException( + "Maximum version {max_version} requested but {version}" + " found".format(max_version=max_version, version=version), + version=version) if service_key == 'database': # TODO(mordred) Remove when https://review.openstack.org/314032 # has landed and released. We're passing in a Session, but the diff --git a/os_client_config/exceptions.py b/os_client_config/exceptions.py index ab78dc2e5..556dd49bc 100644 --- a/os_client_config/exceptions.py +++ b/os_client_config/exceptions.py @@ -15,3 +15,11 @@ class OpenStackConfigException(Exception): """Something went wrong with parsing your OpenStack Config.""" + + +class OpenStackConfigVersionException(OpenStackConfigException): + """A version was requested that is different than what was found.""" + + def __init__(self, version): + super(OpenStackConfigVersionException, self).__init__() + self.version = version diff --git a/releasenotes/notes/min-max-legacy-version-301242466ddefa93.yaml b/releasenotes/notes/min-max-legacy-version-301242466ddefa93.yaml new file mode 100644 index 000000000..30a380225 --- /dev/null +++ b/releasenotes/notes/min-max-legacy-version-301242466ddefa93.yaml @@ -0,0 +1,15 @@ +--- +features: + - Add min_version and max_version to get_legacy_client + and to get_session_endpoint. At the moment this is only + really fully plumbed through for cinder, which has extra + special fun around volume, volumev2 and volumev3. Min and max + versions to both methods will look through the options available + in the service catalog and try to return the latest one available + from the span of requested versions. This means a user can say + volume_api_version=None, min_version=2, max_version=3 will get + an endpoint from get_session_endpoint or a Client from cinderclient + that will be either v2 or v3 but not v1. In the future, min and max + version for get_session_endpoint should be able to sort out + appropriate endpoints via version discovery, but that does not + currently exist. From 38e5eba621e48d74c05315da2b89e6c801f4c43f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Thu, 9 Mar 2017 09:32:15 -0600 Subject: [PATCH 333/365] Use interface not endpoint_type for keystoneclient keystoneclient wants the interface argument. Change-Id: I5898d8621259256f962fc006df38049d0cb059f8 --- os_client_config/cloud_config.py | 2 +- os_client_config/tests/test_cloud_config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 8eae86f77..ae74e5524 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -321,7 +321,7 @@ class CloudConfig(object): endpoint_override = self.get_endpoint(service_key) if not interface_key: - if service_key in ('image', 'key-manager'): + if service_key in ('image', 'key-manager', 'identity'): interface_key = 'interface' else: interface_key = 'endpoint_type' diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index ce724cb56..3c1ae1f34 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -529,7 +529,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='2.0', endpoint='http://example.com/v2', - endpoint_type='admin', + interface='admin', endpoint_override=None, region_name='region-al', service_type='identity', @@ -549,7 +549,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='3', endpoint='http://example.com', - endpoint_type='admin', + interface='admin', endpoint_override=None, region_name='region-al', service_type='identity', From 194e53c84e32d2fde851f03992788d3b95fab3e3 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 13 Mar 2017 11:36:28 -0700 Subject: [PATCH 334/365] OVH supports qcow2 OVH supports qcow2 images too. Update the docs and vendor json file to reflect this. You can continue to use raw images just fine as well. Change-Id: Ic7dc4c70c681947a0475bbabf5621672825dfb3c --- doc/source/vendor-support.rst | 2 +- os_client_config/vendors/ovh.json | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 577093c57..4400e5e46 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -223,7 +223,7 @@ SBG1 Strassbourg, FR GRA1 Gravelines, FR ============== ================ -* Images must be in `raw` format +* Images may be in `raw` format. The `qcow2` default is also supported * Floating IPs are not supported rackspace diff --git a/os_client_config/vendors/ovh.json b/os_client_config/vendors/ovh.json index 664f1617f..f17dc2b68 100644 --- a/os_client_config/vendors/ovh.json +++ b/os_client_config/vendors/ovh.json @@ -10,7 +10,6 @@ "SBG1" ], "identity_api_version": "3", - "image_format": "raw", "floating_ip_source": "None" } } From bd0a40ecc646e9fe1d8bad7eb21abc58b1d9baaa Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 20 Mar 2017 11:19:08 -0500 Subject: [PATCH 335/365] Add designateclient to constructors list We've had this mapping over in shade for a while. No sense in keeping the fun all to ourselves. Change-Id: Icb2b98b621cfa8cff86c534bfba8f5de2c818e95 --- os_client_config/constructors.json | 1 + 1 file changed, 1 insertion(+) diff --git a/os_client_config/constructors.json b/os_client_config/constructors.json index 7ad420e61..2819bf380 100644 --- a/os_client_config/constructors.json +++ b/os_client_config/constructors.json @@ -3,6 +3,7 @@ "compute": "novaclient.client.Client", "container-infra": "magnumclient.client.Client", "database": "troveclient.client.Client", + "dns": "designateclient.client.Client", "identity": "keystoneclient.client.Client", "image": "glanceclient.Client", "key-manager": "barbicanclient.client.Client", From 01ff292e078206e487751228be4a7062ba0c6048 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 27 Mar 2017 09:16:35 -0500 Subject: [PATCH 336/365] Stop special-casing idenity catalog lookups We have a special case to work around a thing we're pretty sure keystoneclient used to do but apparently doesn't do anymore. Remove the workaround. Co-Authored-By: Jamie Lennox Change-Id: I873ad91816150b593d4aef13dcd1520e8c91b22a --- os_client_config/cloud_config.py | 42 +++++++++------------ os_client_config/tests/test_cloud_config.py | 13 ------- 2 files changed, 17 insertions(+), 38 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 99ef2ba73..3ba3541ea 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -18,7 +18,6 @@ import warnings from keystoneauth1 import adapter import keystoneauth1.exceptions.catalog -from keystoneauth1 import plugin from keystoneauth1 import session import requestsexceptions @@ -272,31 +271,24 @@ class CloudConfig(object): 'service_name': self.get_service_name(service_key), 'region_name': self.region } - if service_key == 'identity': - # setting interface in kwargs dict even though we don't use kwargs - # dict here just for ease of warning text later - kwargs['interface'] = plugin.AUTH_INTERFACE - session = self.get_session() - endpoint = session.get_endpoint(interface=kwargs['interface']) + kwargs['interface'] = self.get_interface(service_key) + if service_key == 'volume' and not self.get_api_version('volume'): + # If we don't have a configured cinder version, we can't know + # to request a different service_type + min_version = float(min_version or 1) + max_version = float(max_version or 3) + min_major = math.trunc(float(min_version)) + max_major = math.trunc(float(max_version)) + versions = range(int(max_major) + 1, int(min_major), -1) + service_types = [] + for version in versions: + if version == 1: + service_types.append('volume') + else: + service_types.append('volumev{v}'.format(v=version)) else: - kwargs['interface'] = self.get_interface(service_key) - if service_key == 'volume' and not self.get_api_version('volume'): - # If we don't have a configured cinder version, we can't know - # to request a different service_type - min_version = float(min_version or 1) - max_version = float(max_version or 3) - min_major = math.trunc(float(min_version)) - max_major = math.trunc(float(max_version)) - versions = range(int(max_major) + 1, int(min_major), -1) - service_types = [] - for version in versions: - if version == 1: - service_types.append('volume') - else: - service_types.append('volumev{v}'.format(v=version)) - else: - service_types = [self.get_service_type(service_key)] - endpoint = self._get_highest_endpoint(service_types, kwargs) + service_types = [self.get_service_type(service_key)] + endpoint = self._get_highest_endpoint(service_types, kwargs) if not endpoint: self.log.warning( "Keystone catalog entry not found (" diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 3c1ae1f34..cb6d91c3d 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -13,7 +13,6 @@ import copy from keystoneauth1 import exceptions as ksa_exceptions -from keystoneauth1 import plugin as ksa_plugin from keystoneauth1 import session as ksa_session import mock @@ -219,18 +218,6 @@ class TestCloudConfig(base.TestCase): cc.get_session_endpoint('telemetry'), fake_services_dict['telemetry_endpoint']) - @mock.patch.object(cloud_config.CloudConfig, 'get_session') - def test_session_endpoint_identity(self, mock_get_session): - mock_session = mock.Mock() - mock_get_session.return_value = mock_session - config_dict = defaults.get_defaults() - config_dict.update(fake_services_dict) - cc = cloud_config.CloudConfig( - "test1", "region-al", config_dict, auth_plugin=mock.Mock()) - cc.get_session_endpoint('identity') - mock_session.get_endpoint.assert_called_with( - interface=ksa_plugin.AUTH_INTERFACE) - @mock.patch.object(cloud_config.CloudConfig, 'get_session') def test_session_endpoint(self, mock_get_session): mock_session = mock.Mock() From 451ec8daadfb6702841878cac1a8d4e6012b838d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 28 Mar 2017 10:59:32 -0500 Subject: [PATCH 337/365] Remove out of date comment Change-Id: I8a26f5952456a96429ff1413b90aef3091a8b5bf --- os_client_config/cloud_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 3ba3541ea..45bb82599 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -265,7 +265,6 @@ class CloudConfig(object): override_endpoint = self.get_endpoint(service_key) if override_endpoint: return override_endpoint - # keystone is a special case in keystone, because what? endpoint = None kwargs = { 'service_name': self.get_service_name(service_key), From b31e9aa777a028e34814499d6821ac1c1bfbb881 Mon Sep 17 00:00:00 2001 From: Jim Rollenhagen Date: Wed, 29 Mar 2017 16:54:30 -0400 Subject: [PATCH 338/365] Docs: add a note about rackspace API keys Some users are forced to use these instead of passwords (whether because they use 2-factor auth or by policy). Document it so they know how. Change-Id: I558c2e8d3e8b0fad0a96a361232f14443e82a35f --- doc/source/vendor-support.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/vendor-support.rst b/doc/source/vendor-support.rst index 4400e5e46..449fc5af8 100644 --- a/doc/source/vendor-support.rst +++ b/doc/source/vendor-support.rst @@ -254,6 +254,14 @@ SYD Sydney, NSW :vm_mode: hvm :xenapi_use_agent: False * Volume API Version is 1 +* While passwords are recommended for use, API keys do work as well. + The `rackspaceauth` python package must be installed, and then the following + can be added to clouds.yaml:: + + auth: + username: myusername + api_key: myapikey + auth_type: rackspace_apikey switchengines ------------- From 64b28d42eda595a6fb4ee8b46d93cd61e612aae1 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 2 Apr 2017 11:22:34 -0500 Subject: [PATCH 339/365] Add ability to pass in user_agent keystoneauth supports adding a user_agent info to the Session and Adapter via app_name. Allow users to add app_name/app_name and versions as desired. Also, add os-client-config into additional_user_agent. As an example, once this is landed and plumbed through shade, nodepool will set app_name='nodepool' and we'll have: User-Agent: nodepool/0.4.0 os-client-config/1.26.1 shade/1.19.1 keystoneauth1/2.18.0 python-requests/2.13.0 CPython/2.7.12 Change-Id: I1eb4dbd2587dcbe297b5c060c3c34b68ef51ef5e --- os_client_config/__init__.py | 19 +++++++++++++++---- os_client_config/cloud_config.py | 11 ++++++++++- os_client_config/config.py | 5 +++++ os_client_config/tests/base.py | 2 ++ os_client_config/tests/test_cloud_config.py | 18 ++++++++++++++++-- 5 files changed, 48 insertions(+), 7 deletions(-) diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index a36a13061..1f1266ce1 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -23,9 +23,14 @@ from os_client_config.config import OpenStackConfig # noqa __version__ = pbr.version.VersionInfo('os_client_config').version_string() -def get_config(service_key=None, options=None, **kwargs): +def get_config( + service_key=None, options=None, + app_name=None, app_version=None, + **kwargs): load_yaml_config = kwargs.pop('load_yaml_config', True) - config = OpenStackConfig(load_yaml_config=load_yaml_config) + config = OpenStackConfig( + load_yaml_config=load_yaml_config, + app_name=app_name, app_version=app_version) if options: config.register_argparse_arguments(options, sys.argv, service_key) parsed_options = options.parse_known_args(sys.argv) @@ -35,7 +40,10 @@ def get_config(service_key=None, options=None, **kwargs): return config.get_one_cloud(options=parsed_options, **kwargs) -def make_rest_client(service_key, options=None, **kwargs): +def make_rest_client( + service_key, options=None, + app_name=None, app_version=None, + **kwargs): """Simple wrapper function. It has almost no features. This will get you a raw requests Session Adapter that is mounted @@ -48,7 +56,10 @@ def make_rest_client(service_key, options=None, **kwargs): get_session_client on it. This function is to make it easy to poke at OpenStack REST APIs with a properly configured keystone session. """ - cloud = get_config(service_key=service_key, options=options, **kwargs) + cloud = get_config( + service_key=service_key, options=options, + app_name=app_name, app_version=app_version, + **kwargs) return cloud.get_session_client(service_key) # Backwards compat - simple_client was a terrible name simple_client = make_rest_client diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 45bb82599..086092488 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -21,6 +21,7 @@ import keystoneauth1.exceptions.catalog from keystoneauth1 import session import requestsexceptions +import os_client_config from os_client_config import _log from os_client_config import constructors from os_client_config import exceptions @@ -71,7 +72,8 @@ def _make_key(key, service_type): class CloudConfig(object): def __init__(self, name, region, config, force_ipv4=False, auth_plugin=None, - openstack_config=None, session_constructor=None): + openstack_config=None, session_constructor=None, + app_name=None, app_version=None): self.name = name self.region = region self.config = config @@ -81,6 +83,8 @@ class CloudConfig(object): self._openstack_config = openstack_config self._keystone_session = None self._session_constructor = session_constructor or session.Session + self._app_name = app_name + self._app_version = app_version def __getattr__(self, key): """Return arbitrary attributes.""" @@ -211,9 +215,14 @@ class CloudConfig(object): requestsexceptions.squelch_warnings(insecure_requests=not verify) self._keystone_session = self._session_constructor( auth=self._auth, + app_name=self._app_name, + app_version=self._app_version, verify=verify, cert=cert, timeout=self.config['api_timeout']) + if hasattr(self._keystone_session, 'additional_user_agent'): + self._keystone_session.additional_user_agent.append( + ('os-client-config', os_client_config.__version__)) return self._keystone_session def get_session_client(self, service_key): diff --git a/os_client_config/config.py b/os_client_config/config.py index 64e3a13e6..89b5c6ccf 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -174,9 +174,12 @@ class OpenStackConfig(object): override_defaults=None, force_ipv4=None, envvar_prefix=None, secure_files=None, pw_func=None, session_constructor=None, + app_name=None, app_version=None, load_yaml_config=True): self.log = _log.setup_logging(__name__) self._session_constructor = session_constructor + self._app_name = app_name + self._app_version = app_version if load_yaml_config: self._config_files = config_files or CONFIG_FILES @@ -1088,6 +1091,8 @@ class OpenStackConfig(object): auth_plugin=auth_plugin, openstack_config=self, session_constructor=self._session_constructor, + app_name=self._app_name, + app_version=self._app_version, ) def get_one_cloud_osc( diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 85a42c593..9710782d4 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -212,6 +212,8 @@ class TestCase(base.BaseTestCase): self.secure_yaml = _write_yaml(SECURE_CONF) self.vendor_yaml = _write_yaml(VENDOR_CONF) self.no_yaml = _write_yaml(NO_CONF) + self.useFixture(fixtures.MonkeyPatch( + 'os_client_config.__version__', '1.2.3')) # Isolate the test runs from the environment # Do this as two loops because you can't modify the dict in a loop diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index cb6d91c3d..0671fcc8e 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -179,15 +179,25 @@ class TestCloudConfig(base.TestCase): def test_get_session(self, mock_session): config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) + fake_session = mock.Mock() + fake_session.additional_user_agent = [] + mock_session.return_value = fake_session cc = cloud_config.CloudConfig( "test1", "region-al", config_dict, auth_plugin=mock.Mock()) cc.get_session() mock_session.assert_called_with( auth=mock.ANY, - verify=True, cert=None, timeout=None) + verify=True, cert=None, timeout=None, + app_name=None, app_version=None) + self.assertEqual( + fake_session.additional_user_agent, + [('os-client-config', '1.2.3')]) @mock.patch.object(ksa_session, 'Session') def test_get_session_with_timeout(self, mock_session): + fake_session = mock.Mock() + fake_session.additional_user_agent = [] + mock_session.return_value = fake_session config_dict = defaults.get_defaults() config_dict.update(fake_services_dict) config_dict['api_timeout'] = 9 @@ -196,7 +206,11 @@ class TestCloudConfig(base.TestCase): cc.get_session() mock_session.assert_called_with( auth=mock.ANY, - verify=True, cert=None, timeout=9) + verify=True, cert=None, timeout=9, + app_name=None, app_version=None) + self.assertEqual( + fake_session.additional_user_agent, + [('os-client-config', '1.2.3')]) @mock.patch.object(ksa_session, 'Session') def test_override_session_endpoint_override(self, mock_session): From ff2c06c30538edb9cb47b83f0a194f7893a7f458 Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Sun, 23 Apr 2017 17:42:47 +0200 Subject: [PATCH 340/365] Make _fix_argv() somewhat compatible with Argparse action='append' Python Argparse supports the 'append' action [1] which is super handy to allow a user to repeat several times the same argument, each time with different values. This doesn't work with occ that tries to "fix argv" but raises this error: os_client_config.exceptions.OpenStackConfigException: The following options were given: '--foo,--foo' which contain duplicates except that one has _ and one has -. There is no sane way for us to know what you're doing. Remove the duplicate option and try again This patch tweak the _fix_argv() function so that it doesn't explode if the duplicate option has no '_' not '-' in its name. Change-Id: I4f06b6aff8d3ab1df45637399bc3a9b4b61764a9 Related-bug: #1685630 --- os_client_config/config.py | 5 ++++- os_client_config/tests/test_config.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 89b5c6ccf..96d7f5355 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -19,6 +19,7 @@ import collections import copy import json import os +import re import sys import warnings @@ -146,7 +147,9 @@ def _fix_argv(argv): # over the place. processed = collections.defaultdict(list) for index in range(0, len(argv)): - if argv[index].startswith('--'): + # If the value starts with '--' and has '-' or '_' in it, then + # it's worth looking at it + if re.match('^--.*(_|-)+.*', argv[index]): split_args = argv[index].split('=') orig = split_args[0] new = orig.replace('_', '-') diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index dcc841ae9..09a11d28f 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -702,6 +702,17 @@ class TestConfigArgparse(base.TestCase): self.assertEqual(cc.config['auth']['password'], 'pass') self.assertEqual(cc.config['auth']['auth_url'], 'auth-url') + def test_argparse_action_append_no_underscore(self): + c = config.OpenStackConfig(config_files=[self.no_yaml], + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) + parser = argparse.ArgumentParser() + parser.add_argument('--foo', action='append') + argv = ['--foo', '1', '--foo', '2'] + c.register_argparse_arguments(parser, argv=argv) + opts, _remain = parser.parse_known_args(argv) + self.assertEqual(opts.foo, ['1', '2']) + def test_argparse_underscores_duplicate(self): c = config.OpenStackConfig(config_files=[self.no_yaml], vendor_files=[self.no_yaml], From 17debbb09907b83d5781ad683f4b72c9eb1ce221 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 28 Apr 2017 12:46:44 -0500 Subject: [PATCH 341/365] Fix interactions with keystoneauth from newton keystoneauth in newton did not have app_name or app_version as Session parameters. Although it isn't a super common combination, user agent strings aren't a reason to break something. Add a simple workaround. Change-Id: Ib5774389fefdbc190a4b78dd6784c8006afbb270 --- os_client_config/cloud_config.py | 10 +++++-- os_client_config/tests/test_cloud_config.py | 28 ++++++++++++++++--- ...ith-old-keystoneauth-66e11ee9d008b962.yaml | 7 +++++ 3 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/fix-compat-with-old-keystoneauth-66e11ee9d008b962.yaml diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 086092488..fec350094 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -215,14 +215,20 @@ class CloudConfig(object): requestsexceptions.squelch_warnings(insecure_requests=not verify) self._keystone_session = self._session_constructor( auth=self._auth, - app_name=self._app_name, - app_version=self._app_version, verify=verify, cert=cert, timeout=self.config['api_timeout']) if hasattr(self._keystone_session, 'additional_user_agent'): self._keystone_session.additional_user_agent.append( ('os-client-config', os_client_config.__version__)) + # Using old keystoneauth with new os-client-config fails if + # we pass in app_name and app_version. Those are not essential, + # nor a reason to bump our minimum, so just test for the session + # having the attribute post creation and set them then. + if hasattr(self._keystone_session, 'app_name'): + self._keystone_session.app_name = self._app_name + if hasattr(self._keystone_session, 'app_version'): + self._keystone_session.app_version = self._app_version return self._keystone_session def get_session_client(self, service_key): diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 0671fcc8e..5df1fafa2 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -187,8 +187,29 @@ class TestCloudConfig(base.TestCase): cc.get_session() mock_session.assert_called_with( auth=mock.ANY, - verify=True, cert=None, timeout=None, - app_name=None, app_version=None) + verify=True, cert=None, timeout=None) + self.assertEqual( + fake_session.additional_user_agent, + [('os-client-config', '1.2.3')]) + + @mock.patch.object(ksa_session, 'Session') + def test_get_session_with_app_name(self, mock_session): + config_dict = defaults.get_defaults() + config_dict.update(fake_services_dict) + fake_session = mock.Mock() + fake_session.additional_user_agent = [] + fake_session.app_name = None + fake_session.app_version = None + mock_session.return_value = fake_session + cc = cloud_config.CloudConfig( + "test1", "region-al", config_dict, auth_plugin=mock.Mock(), + app_name="test_app", app_version="test_version") + cc.get_session() + mock_session.assert_called_with( + auth=mock.ANY, + verify=True, cert=None, timeout=None) + self.assertEqual(fake_session.app_name, "test_app") + self.assertEqual(fake_session.app_version, "test_version") self.assertEqual( fake_session.additional_user_agent, [('os-client-config', '1.2.3')]) @@ -206,8 +227,7 @@ class TestCloudConfig(base.TestCase): cc.get_session() mock_session.assert_called_with( auth=mock.ANY, - verify=True, cert=None, timeout=9, - app_name=None, app_version=None) + verify=True, cert=None, timeout=9) self.assertEqual( fake_session.additional_user_agent, [('os-client-config', '1.2.3')]) diff --git a/releasenotes/notes/fix-compat-with-old-keystoneauth-66e11ee9d008b962.yaml b/releasenotes/notes/fix-compat-with-old-keystoneauth-66e11ee9d008b962.yaml new file mode 100644 index 000000000..80d09fb83 --- /dev/null +++ b/releasenotes/notes/fix-compat-with-old-keystoneauth-66e11ee9d008b962.yaml @@ -0,0 +1,7 @@ +--- +issues: + - Fixed a regression when using latest os-client-config with + the keystoneauth from stable/newton. Although this isn't a + super common combination, the added feature that broke the + interaction is really not worthy of the incompatibility, so + a workaround was added. From 347fe82c929bb9b8aba5aee504bbe3eaf1e8bf04 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 8 May 2017 07:20:53 -0400 Subject: [PATCH 342/365] Add helper method to fetch service catalog Grabbing the catalog is weird. OCC should help. Change-Id: I6e7176568311c1f0e644a8e8876f56c3e153d6e8 --- os_client_config/cloud_config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 086092488..17cc801e2 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -225,6 +225,10 @@ class CloudConfig(object): ('os-client-config', os_client_config.__version__)) return self._keystone_session + def get_service_catalog(self): + """Helper method to grab the service catalog.""" + return self._auth.get_access(self.get_session()).service_catalog + def get_session_client(self, service_key): """Return a prepped requests adapter for a given service. From 7a3acde3ed8a68acc15596be2d297426ee357084 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 12 May 2017 12:05:50 -0500 Subject: [PATCH 343/365] Add ironicclient to constructors list Change-Id: I29db3c830759a80d8ea9f0d93a213b4bae4c8b59 --- os_client_config/constructors.json | 1 + 1 file changed, 1 insertion(+) diff --git a/os_client_config/constructors.json b/os_client_config/constructors.json index 2819bf380..9acb7cfb9 100644 --- a/os_client_config/constructors.json +++ b/os_client_config/constructors.json @@ -1,5 +1,6 @@ { "application-catalog": "muranoclient.client.Client", + "baremetal": "ironicclient.client.Client", "compute": "novaclient.client.Client", "container-infra": "magnumclient.client.Client", "database": "troveclient.client.Client", From 4493871824839782846c3825c754895876f8d08e Mon Sep 17 00:00:00 2001 From: Jordan Pittier Date: Wed, 3 May 2017 13:11:10 +0000 Subject: [PATCH 344/365] Revert "Use interface not endpoint_type for keystoneclient" This reverts commit 38e5eba621e48d74c05315da2b89e6c801f4c43f. This patch introduced a bug when using Keystone v2. With this patch, the following works: python -c "import os_client_config; print(os_client_config.make_client('identity', auth_url='http://localhost/identity_admin', username='admin', project_name='admin', password='testtest', identity_api_version='3').roles.list())" But changing identity_api_version from 3 to 2.0 raises an exception. Without this patch, both 3 and 2.0 works. Change-Id: I8d2ad71ff51a08af1166d36805b740ea272939ed --- os_client_config/cloud_config.py | 2 +- os_client_config/tests/test_cloud_config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index fec350094..776ea8a17 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -367,7 +367,7 @@ class CloudConfig(object): endpoint_override = self.get_endpoint(service_key) if not interface_key: - if service_key in ('image', 'key-manager', 'identity'): + if service_key in ('image', 'key-manager'): interface_key = 'interface' else: interface_key = 'endpoint_type' diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 5df1fafa2..3ec7ecf88 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -550,7 +550,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='2.0', endpoint='http://example.com/v2', - interface='admin', + endpoint_type='admin', endpoint_override=None, region_name='region-al', service_type='identity', @@ -570,7 +570,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='3', endpoint='http://example.com', - interface='admin', + endpoint_type='admin', endpoint_override=None, region_name='region-al', service_type='identity', From a483534123640eca2bcbf7c06920be3a2a01b090 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 13 May 2017 08:45:26 -0500 Subject: [PATCH 345/365] Revert "Revert "Use interface not endpoint_type for keystoneclient"" Unrevert the endpoint_type/interface patch. But this time around, put in a check for API version 2.0 and only apply the interface arg if it's for v3. This reverts commit 4493871824839782846c3825c754895876f8d08e. Change-Id: Ib347ec686d4d01788ee943c4c4f809aad06d9ccf --- os_client_config/cloud_config.py | 23 ++++++++++++--------- os_client_config/tests/test_cloud_config.py | 2 +- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 776ea8a17..89e070f39 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -366,12 +366,6 @@ class CloudConfig(object): service_key, min_version=min_version, max_version=max_version) endpoint_override = self.get_endpoint(service_key) - if not interface_key: - if service_key in ('image', 'key-manager'): - interface_key = 'interface' - else: - interface_key = 'endpoint_type' - if service_key == 'object-store': constructor_kwargs = dict( session=self.get_session(), @@ -404,10 +398,6 @@ class CloudConfig(object): if not endpoint_override: constructor_kwargs['endpoint_override'] = endpoint constructor_kwargs.update(kwargs) - if service_key == 'object-store': - constructor_kwargs['os_options'][interface_key] = interface - else: - constructor_kwargs[interface_key] = interface if pass_version_arg and service_key != 'object-store': if not version: version = self.get_api_version(service_key) @@ -451,6 +441,19 @@ class CloudConfig(object): constructor_kwargs['username'] = None constructor_kwargs['password'] = None + if not interface_key: + if service_key in ('image', 'key-manager'): + interface_key = 'interface' + elif (service_key == 'identity' + and version and version.startswith('3')): + interface_key = 'interface' + else: + interface_key = 'endpoint_type' + if service_key == 'object-store': + constructor_kwargs['os_options'][interface_key] = interface + else: + constructor_kwargs[interface_key] = interface + return client_class(**constructor_kwargs) def get_cache_expiration_time(self): diff --git a/os_client_config/tests/test_cloud_config.py b/os_client_config/tests/test_cloud_config.py index 3ec7ecf88..813552639 100644 --- a/os_client_config/tests/test_cloud_config.py +++ b/os_client_config/tests/test_cloud_config.py @@ -570,7 +570,7 @@ class TestCloudConfig(base.TestCase): mock_client.assert_called_with( version='3', endpoint='http://example.com', - endpoint_type='admin', + interface='admin', endpoint_override=None, region_name='region-al', service_type='identity', From 8235e0ce6cff00bfa84501865a3cf48511cdf2ec Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 16 May 2017 18:12:41 -0500 Subject: [PATCH 346/365] Keep a singleton to support multiple get_config calls We are destructive to os.environ in the OpenStackConfig constructor- so it really should only ever be called once. Make sure get_config does this. Change-Id: I279bdf68408a807ec18fba634df3769c9b8fc4dc Closes-Bug: #1691294 --- os_client_config/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/os_client_config/__init__.py b/os_client_config/__init__.py index 1f1266ce1..424652541 100644 --- a/os_client_config/__init__.py +++ b/os_client_config/__init__.py @@ -21,6 +21,7 @@ from os_client_config.config import OpenStackConfig # noqa __version__ = pbr.version.VersionInfo('os_client_config').version_string() +_config = None def get_config( @@ -28,16 +29,18 @@ def get_config( app_name=None, app_version=None, **kwargs): load_yaml_config = kwargs.pop('load_yaml_config', True) - config = OpenStackConfig( - load_yaml_config=load_yaml_config, - app_name=app_name, app_version=app_version) + global _config + if not _config: + _config = OpenStackConfig( + load_yaml_config=load_yaml_config, + app_name=app_name, app_version=app_version) if options: - config.register_argparse_arguments(options, sys.argv, service_key) + _config.register_argparse_arguments(options, sys.argv, service_key) parsed_options = options.parse_known_args(sys.argv) else: parsed_options = None - return config.get_one_cloud(options=parsed_options, **kwargs) + return _config.get_one_cloud(options=parsed_options, **kwargs) def make_rest_client( From 990cfa3ce24d0a92c7578b52ae1fea02d69f7e87 Mon Sep 17 00:00:00 2001 From: Matthew Booth Date: Thu, 25 May 2017 13:26:48 +0100 Subject: [PATCH 347/365] Don't pop from os.environ It's rude to other users and subsequent callers. Change-Id: I7789f381c99311bfd1c1e0a9869cbacbc96b17d6 --- os_client_config/config.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/os_client_config/config.py b/os_client_config/config.py index 96d7f5355..1ed416cf5 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -106,9 +106,12 @@ def _get_os_environ(envvar_prefix=None): for k in environkeys: newkey = k.split('_', 1)[-1].lower() ret[newkey] = os.environ[k] - # If the only environ keys are cloud and region_name, don't return anything - # because they are cloud selectors - if set(environkeys) - set(['OS_CLOUD', 'OS_REGION_NAME']): + # If the only environ keys are selectors or behavior modification, don't + # return anything + selectors = set([ + 'OS_CLOUD', 'OS_REGION_NAME', + 'OS_CLIENT_CONFIG_FILE', 'OS_CLIENT_SECURE_FILE', 'OS_CLOUD_NAME']) + if set(environkeys) - selectors: return ret return None @@ -193,11 +196,11 @@ class OpenStackConfig(object): self._secure_files = [] self._vendor_files = [] - config_file_override = os.environ.pop('OS_CLIENT_CONFIG_FILE', None) + config_file_override = os.environ.get('OS_CLIENT_CONFIG_FILE') if config_file_override: self._config_files.insert(0, config_file_override) - secure_file_override = os.environ.pop('OS_CLIENT_SECURE_FILE', None) + secure_file_override = os.environ.get('OS_CLIENT_SECURE_FILE') if secure_file_override: self._secure_files.insert(0, secure_file_override) @@ -226,12 +229,12 @@ class OpenStackConfig(object): else: # Get the backwards compat value prefer_ipv6 = get_boolean( - os.environ.pop( + os.environ.get( 'OS_PREFER_IPV6', client_config.get( 'prefer_ipv6', client_config.get( 'prefer-ipv6', True)))) force_ipv4 = get_boolean( - os.environ.pop( + os.environ.get( 'OS_FORCE_IPV4', client_config.get( 'force_ipv4', client_config.get( 'broken-ipv6', False)))) @@ -243,7 +246,7 @@ class OpenStackConfig(object): self.force_ipv4 = True # Next, process environment variables and add them to the mix - self.envvar_key = os.environ.pop('OS_CLOUD_NAME', 'envvars') + self.envvar_key = os.environ.get('OS_CLOUD_NAME', 'envvars') if self.envvar_key in self.cloud_config['clouds']: raise exceptions.OpenStackConfigException( '"{0}" defines a cloud named "{1}", but' @@ -251,9 +254,8 @@ class OpenStackConfig(object): ' either your environment based cloud, or one of your' ' file-based clouds.'.format(self.config_filename, self.envvar_key)) - # Pull out OS_CLOUD so that if it's the only thing set, do not - # make an envvars cloud - self.default_cloud = os.environ.pop('OS_CLOUD', None) + + self.default_cloud = os.environ.get('OS_CLOUD') envvars = _get_os_environ(envvar_prefix=envvar_prefix) if envvars: From 95f44f171c614cb0bb1650eb0062baea5c96f25d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Jun 2017 15:15:52 -0400 Subject: [PATCH 348/365] rearrange existing documentation to follow the new standard layout Depends-On: Ia750cb049c0f53a234ea70ce1f2bbbb7a2aa9454 Change-Id: Ib68812eda4ab0ce09c6d438f9f236111d61ef38b Signed-off-by: Doug Hellmann --- README.rst | 481 +----------------- .../index.rst} | 2 +- doc/source/index.rst | 28 +- .../{installation.rst => install/index.rst} | 0 .../index.rst} | 0 doc/source/user/configuration.rst | 303 +++++++++++ doc/source/user/index.rst | 12 + doc/source/{ => user}/network-config.rst | 0 doc/source/{ => user}/releasenotes.rst | 0 doc/source/user/using.rst | 182 +++++++ doc/source/{ => user}/vendor-support.rst | 0 11 files changed, 520 insertions(+), 488 deletions(-) rename doc/source/{contributing.rst => contributor/index.rst} (50%) rename doc/source/{installation.rst => install/index.rst} (100%) rename doc/source/{api-reference.rst => reference/index.rst} (100%) create mode 100644 doc/source/user/configuration.rst create mode 100644 doc/source/user/index.rst rename doc/source/{ => user}/network-config.rst (100%) rename doc/source/{ => user}/releasenotes.rst (100%) create mode 100644 doc/source/user/using.rst rename doc/source/{ => user}/vendor-support.rst (100%) diff --git a/README.rst b/README.rst index 67aa91a34..35ff07b43 100644 --- a/README.rst +++ b/README.rst @@ -16,489 +16,10 @@ have to know extra info to use OpenStack * If you have environment variables, you will get a cloud named `envvars` * If you have neither, you will get a cloud named `defaults` with base defaults -Environment Variables ---------------------- - -`os-client-config` honors all of the normal `OS_*` variables. It does not -provide backwards compatibility to service-specific variables such as -`NOVA_USERNAME`. - -If you have OpenStack environment variables set, `os-client-config` will produce -a cloud config object named `envvars` containing your values from the -environment. If you don't like the name `envvars`, that's ok, you can override -it by setting `OS_CLOUD_NAME`. - -Service specific settings, like the nova service type, are set with the -default service type as a prefix. For instance, to set a special service_type -for trove set - -.. code-block:: bash - - export OS_DATABASE_SERVICE_TYPE=rax:database - -Config Files ------------- - -`os-client-config` will look for a file called `clouds.yaml` in the following -locations: - -* Current Directory -* ~/.config/openstack -* /etc/openstack - -The first file found wins. - -You can also set the environment variable `OS_CLIENT_CONFIG_FILE` to an -absolute path of a file to look for and that location will be inserted at the -front of the file search list. - -The keys are all of the keys you'd expect from `OS_*` - except lower case -and without the OS prefix. So, region name is set with `region_name`. - -Service specific settings, like the nova service type, are set with the -default service type as a prefix. For instance, to set a special service_type -for trove (because you're using Rackspace) set: - -.. code-block:: yaml - - database_service_type: 'rax:database' - - -Site Specific File Locations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In addition to `~/.config/openstack` and `/etc/openstack` - some platforms -have other locations they like to put things. `os-client-config` will also -look in an OS specific config dir - -* `USER_CONFIG_DIR` -* `SITE_CONFIG_DIR` - -`USER_CONFIG_DIR` is different on Linux, OSX and Windows. - -* Linux: `~/.config/openstack` -* OSX: `~/Library/Application Support/openstack` -* Windows: `C:\\Users\\USERNAME\\AppData\\Local\\OpenStack\\openstack` - -`SITE_CONFIG_DIR` is different on Linux, OSX and Windows. - -* Linux: `/etc/openstack` -* OSX: `/Library/Application Support/openstack` -* Windows: `C:\\ProgramData\\OpenStack\\openstack` - -An example config file is probably helpful: - -.. code-block:: yaml - - clouds: - mtvexx: - profile: vexxhost - auth: - username: mordred@inaugust.com - password: XXXXXXXXX - project_name: mordred@inaugust.com - region_name: ca-ymq-1 - dns_api_version: 1 - mordred: - region_name: RegionOne - auth: - username: 'mordred' - password: XXXXXXX - project_name: 'shade' - auth_url: 'https://montytaylor-sjc.openstack.blueboxgrid.com:5001/v2.0' - infra: - profile: rackspace - auth: - username: openstackci - password: XXXXXXXX - project_id: 610275 - regions: - - DFW - - ORD - - IAD - -You may note a few things. First, since `auth_url` settings are silly -and embarrassingly ugly, known cloud vendor profile information is included and -may be referenced by name. One of the benefits of that is that `auth_url` -isn't the only thing the vendor defaults contain. For instance, since -Rackspace lists `rax:database` as the service type for trove, `os-client-config` -knows that so that you don't have to. In case the cloud vendor profile is not -available, you can provide one called `clouds-public.yaml`, following the same -location rules previously mentioned for the config files. - -`regions` can be a list of regions. When you call `get_all_clouds`, -you'll get a cloud config object for each cloud/region combo. - -As seen with `dns_service_type`, any setting that makes sense to be per-service, -like `service_type` or `endpoint` or `api_version` can be set by prefixing -the setting with the default service type. That might strike you funny when -setting `service_type` and it does me too - but that's just the world we live -in. - -Auth Settings -------------- - -Keystone has auth plugins - which means it's not possible to know ahead of time -which auth settings are needed. `os-client-config` sets the default plugin type -to `password`, which is what things all were before plugins came about. In -order to facilitate validation of values, all of the parameters that exist -as a result of a chosen plugin need to go into the auth dict. For password -auth, this includes `auth_url`, `username` and `password` as well as anything -related to domains, projects and trusts. - -Splitting Secrets ------------------ - -In some scenarios, such as configuration management controlled environments, -it might be easier to have secrets in one file and non-secrets in another. -This is fully supported via an optional file `secure.yaml` which follows all -the same location rules as `clouds.yaml`. It can contain anything you put -in `clouds.yaml` and will take precedence over anything in the `clouds.yaml` -file. - -.. code-block:: yaml - - # clouds.yaml - clouds: - internap: - profile: internap - auth: - username: api-55f9a00fb2619 - project_name: inap-17037 - regions: - - ams01 - - nyj01 - # secure.yaml - clouds: - internap: - auth: - password: XXXXXXXXXXXXXXXXX - -SSL Settings ------------- - -When the access to a cloud is done via a secure connection, `os-client-config` -will always verify the SSL cert by default. This can be disabled by setting -`verify` to `False`. In case the cert is signed by an unknown CA, a specific -cacert can be provided via `cacert`. **WARNING:** `verify` will always have -precedence over `cacert`, so when setting a CA cert but disabling `verify`, the -cloud cert will never be validated. - -Client certs are also configurable. `cert` will be the client cert file -location. In case the cert key is not included within the client cert file, -its file location needs to be set via `key`. - -.. code-block:: yaml - - # clouds.yaml - clouds: - secure: - auth: ... - key: /home/myhome/client-cert.key - cert: /home/myhome/client-cert.crt - cacert: /home/myhome/ca.crt - insecure: - auth: ... - verify: False - -Cache Settings --------------- - -Accessing a cloud is often expensive, so it's quite common to want to do some -client-side caching of those operations. To facilitate that, `os-client-config` -understands passing through cache settings to dogpile.cache, with the following -behaviors: - -* Listing no config settings means you get a null cache. -* `cache.expiration_time` and nothing else gets you memory cache. -* Otherwise, `cache.class` and `cache.arguments` are passed in - -Different cloud behaviors are also differently expensive to deal with. If you -want to get really crazy and tweak stuff, you can specify different expiration -times on a per-resource basis by passing values, in seconds to an expiration -mapping keyed on the singular name of the resource. A value of `-1` indicates -that the resource should never expire. - -`os-client-config` does not actually cache anything itself, but it collects -and presents the cache information so that your various applications that -are connecting to OpenStack can share a cache should you desire. - -.. code-block:: yaml - - cache: - class: dogpile.cache.pylibmc - expiration_time: 3600 - arguments: - url: - - 127.0.0.1 - expiration: - server: 5 - flavor: -1 - clouds: - mtvexx: - profile: vexxhost - auth: - username: mordred@inaugust.com - password: XXXXXXXXX - project_name: mordred@inaugust.com - region_name: ca-ymq-1 - dns_api_version: 1 - - -IPv6 ----- - -IPv6 is the future, and you should always use it if your cloud supports it and -if your local network supports it. Both of those are easily detectable and all -friendly software should do the right thing. However, sometimes you might -exist in a location where you have an IPv6 stack, but something evil has -caused it to not actually function. In that case, there is a config option -you can set to unbreak you `force_ipv4`, or `OS_FORCE_IPV4` boolean -environment variable. - -.. code-block:: yaml - - client: - force_ipv4: true - clouds: - mtvexx: - profile: vexxhost - auth: - username: mordred@inaugust.com - password: XXXXXXXXX - project_name: mordred@inaugust.com - region_name: ca-ymq-1 - dns_api_version: 1 - monty: - profile: rax - auth: - username: mordred@inaugust.com - password: XXXXXXXXX - project_name: mordred@inaugust.com - region_name: DFW - -The above snippet will tell client programs to prefer returning an IPv4 -address. - -Per-region settings -------------------- - -Sometimes you have a cloud provider that has config that is common to the -cloud, but also with some things you might want to express on a per-region -basis. For instance, Internap provides a public and private network specific -to the user in each region, and putting the values of those networks into -config can make consuming programs more efficient. - -To support this, the region list can actually be a list of dicts, and any -setting that can be set at the cloud level can be overridden for that -region. - -.. code-block:: yaml - - clouds: - internap: - profile: internap - auth: - password: XXXXXXXXXXXXXXXXX - username: api-55f9a00fb2619 - project_name: inap-17037 - regions: - - name: ams01 - values: - networks: - - name: inap-17037-WAN1654 - routes_externally: true - - name: inap-17037-LAN6745 - - name: nyj01 - values: - networks: - - name: inap-17037-WAN1654 - routes_externally: true - - name: inap-17037-LAN6745 - -Usage ------ - -The simplest and least useful thing you can do is: - -.. code-block:: python - - python -m os_client_config.config - -Which will print out whatever if finds for your config. If you want to use -it from python, which is much more likely what you want to do, things like: - -Get a named cloud. - -.. code-block:: python - - import os_client_config - - cloud_config = os_client_config.OpenStackConfig().get_one_cloud( - 'internap', region_name='ams01') - print(cloud_config.name, cloud_config.region, cloud_config.config) - -Or, get all of the clouds. - -.. code-block:: python - - import os_client_config - - cloud_config = os_client_config.OpenStackConfig().get_all_clouds() - for cloud in cloud_config: - print(cloud.name, cloud.region, cloud.config) - -argparse --------- - -If you're using os-client-config from a program that wants to process -command line options, there is a registration function to register the -arguments that both os-client-config and keystoneauth know how to deal -with - as well as a consumption argument. - -.. code-block:: python - - import argparse - import sys - - import os_client_config - - cloud_config = os_client_config.OpenStackConfig() - parser = argparse.ArgumentParser() - cloud_config.register_argparse_arguments(parser, sys.argv) - - options = parser.parse_args() - - cloud = cloud_config.get_one_cloud(argparse=options) - -Constructing OpenStack SDK object ---------------------------------- - -If what you want to do is get an OpenStack SDK Connection and you want it to -do all the normal things related to clouds.yaml, `OS_` environment variables, -a helper function is provided. The following will get you a fully configured -`openstacksdk` instance. - -.. code-block:: python - - import os_client_config - - sdk = os_client_config.make_sdk() - -If you want to do the same thing but on a named cloud. - -.. code-block:: python - - import os_client_config - - sdk = os_client_config.make_sdk(cloud='mtvexx') - -If you want to do the same thing but also support command line parsing. - -.. code-block:: python - - import argparse - - import os_client_config - - sdk = os_client_config.make_sdk(options=argparse.ArgumentParser()) - -It should be noted that OpenStack SDK has ways to construct itself that allow -for additional flexibility. If the helper function here does not meet your -needs, you should see the `from_config` method of -`openstack.connection.Connection `_ - -Constructing shade objects --------------------------- - -If what you want to do is get a -`shade `_ OpenStackCloud object, a -helper function that honors clouds.yaml and `OS_` environment variables is -provided. The following will get you a fully configured `OpenStackCloud` -instance. - -.. code-block:: python - - import os_client_config - - cloud = os_client_config.make_shade() - -If you want to do the same thing but on a named cloud. - -.. code-block:: python - - import os_client_config - - cloud = os_client_config.make_shade(cloud='mtvexx') - -If you want to do the same thing but also support command line parsing. - -.. code-block:: python - - import argparse - - import os_client_config - - cloud = os_client_config.make_shade(options=argparse.ArgumentParser()) - -Constructing REST API Clients ------------------------------ - -What if you want to make direct REST calls via a Session interface? You're -in luck. A similar interface is available as with `openstacksdk` and `shade`. -The main difference is that you need to specify which service you want to -talk to and `make_rest_client` will return you a keystoneauth Session object -that is mounted on the endpoint for the service you're looking for. - -.. code-block:: python - - import os_client_config - - session = os_client_config.make_rest_client('compute', cloud='vexxhost') - - response = session.get('/servers') - server_list = response.json()['servers'] - -Constructing Legacy Client objects ----------------------------------- - -If you want get an old-style Client object from a python-\*client library, -and you want it to do all the normal things related to clouds.yaml, `OS_` -environment variables, a helper function is also provided. The following -will get you a fully configured `novaclient` instance. - -.. code-block:: python - - import os_client_config - - nova = os_client_config.make_client('compute') - -If you want to do the same thing but on a named cloud. - -.. code-block:: python - - import os_client_config - - nova = os_client_config.make_client('compute', cloud='mtvexx') - -If you want to do the same thing but also support command line parsing. - -.. code-block:: python - - import argparse - - import os_client_config - - nova = os_client_config.make_client( - 'compute', options=argparse.ArgumentParser()) - -If you want to get fancier than that in your python, then the rest of the -API is available to you. But often times, you just want to do the one thing. - Source ------ * Free software: Apache license -* Documentation: http://docs.openstack.org/developer/os-client-config +* Documentation: http://docs.openstack.org/os-client-config/latest * Source: http://git.openstack.org/cgit/openstack/os-client-config * Bugs: http://bugs.launchpad.net/os-client-config diff --git a/doc/source/contributing.rst b/doc/source/contributor/index.rst similarity index 50% rename from doc/source/contributing.rst rename to doc/source/contributor/index.rst index ed77c1262..2aa070771 100644 --- a/doc/source/contributing.rst +++ b/doc/source/contributor/index.rst @@ -1,4 +1,4 @@ ============ Contributing ============ -.. include:: ../../CONTRIBUTING.rst \ No newline at end of file +.. include:: ../../../CONTRIBUTING.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index f7263c9c3..5a407adcc 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,14 +1,28 @@ -.. include:: ../../README.rst +================ +os-client-config +================ + +.. image:: http://governance.openstack.org/badges/os-client-config.svg + :target: http://governance.openstack.org/reference/tags/index.html + +`os-client-config` is a library for collecting client configuration for +using an OpenStack cloud in a consistent and comprehensive manner. It +will find cloud config for as few as 1 cloud and as many as you want to +put in a config file. It will read environment variables and config files, +and it also contains some vendor specific default values so that you don't +have to know extra info to use OpenStack + +* If you have a config file, you will get the clouds listed in it +* If you have environment variables, you will get a cloud named `envvars` +* If you have neither, you will get a cloud named `defaults` with base defaults .. toctree:: :maxdepth: 2 - vendor-support - contributing - installation - network-config - api-reference - releasenotes + install/index + user/index + reference/index + contributor/index Indices and tables ================== diff --git a/doc/source/installation.rst b/doc/source/install/index.rst similarity index 100% rename from doc/source/installation.rst rename to doc/source/install/index.rst diff --git a/doc/source/api-reference.rst b/doc/source/reference/index.rst similarity index 100% rename from doc/source/api-reference.rst rename to doc/source/reference/index.rst diff --git a/doc/source/user/configuration.rst b/doc/source/user/configuration.rst new file mode 100644 index 000000000..df0b26659 --- /dev/null +++ b/doc/source/user/configuration.rst @@ -0,0 +1,303 @@ +=========================================== + Configuring os-client-config Applications +=========================================== + +Environment Variables +--------------------- + +`os-client-config` honors all of the normal `OS_*` variables. It does not +provide backwards compatibility to service-specific variables such as +`NOVA_USERNAME`. + +If you have OpenStack environment variables set, `os-client-config` will produce +a cloud config object named `envvars` containing your values from the +environment. If you don't like the name `envvars`, that's ok, you can override +it by setting `OS_CLOUD_NAME`. + +Service specific settings, like the nova service type, are set with the +default service type as a prefix. For instance, to set a special service_type +for trove set + +.. code-block:: bash + + export OS_DATABASE_SERVICE_TYPE=rax:database + +Config Files +------------ + +`os-client-config` will look for a file called `clouds.yaml` in the following +locations: + +* Current Directory +* ~/.config/openstack +* /etc/openstack + +The first file found wins. + +You can also set the environment variable `OS_CLIENT_CONFIG_FILE` to an +absolute path of a file to look for and that location will be inserted at the +front of the file search list. + +The keys are all of the keys you'd expect from `OS_*` - except lower case +and without the OS prefix. So, region name is set with `region_name`. + +Service specific settings, like the nova service type, are set with the +default service type as a prefix. For instance, to set a special service_type +for trove (because you're using Rackspace) set: + +.. code-block:: yaml + + database_service_type: 'rax:database' + + +Site Specific File Locations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In addition to `~/.config/openstack` and `/etc/openstack` - some platforms +have other locations they like to put things. `os-client-config` will also +look in an OS specific config dir + +* `USER_CONFIG_DIR` +* `SITE_CONFIG_DIR` + +`USER_CONFIG_DIR` is different on Linux, OSX and Windows. + +* Linux: `~/.config/openstack` +* OSX: `~/Library/Application Support/openstack` +* Windows: `C:\\Users\\USERNAME\\AppData\\Local\\OpenStack\\openstack` + +`SITE_CONFIG_DIR` is different on Linux, OSX and Windows. + +* Linux: `/etc/openstack` +* OSX: `/Library/Application Support/openstack` +* Windows: `C:\\ProgramData\\OpenStack\\openstack` + +An example config file is probably helpful: + +.. code-block:: yaml + + clouds: + mtvexx: + profile: vexxhost + auth: + username: mordred@inaugust.com + password: XXXXXXXXX + project_name: mordred@inaugust.com + region_name: ca-ymq-1 + dns_api_version: 1 + mordred: + region_name: RegionOne + auth: + username: 'mordred' + password: XXXXXXX + project_name: 'shade' + auth_url: 'https://montytaylor-sjc.openstack.blueboxgrid.com:5001/v2.0' + infra: + profile: rackspace + auth: + username: openstackci + password: XXXXXXXX + project_id: 610275 + regions: + - DFW + - ORD + - IAD + +You may note a few things. First, since `auth_url` settings are silly +and embarrassingly ugly, known cloud vendor profile information is included and +may be referenced by name. One of the benefits of that is that `auth_url` +isn't the only thing the vendor defaults contain. For instance, since +Rackspace lists `rax:database` as the service type for trove, `os-client-config` +knows that so that you don't have to. In case the cloud vendor profile is not +available, you can provide one called `clouds-public.yaml`, following the same +location rules previously mentioned for the config files. + +`regions` can be a list of regions. When you call `get_all_clouds`, +you'll get a cloud config object for each cloud/region combo. + +As seen with `dns_service_type`, any setting that makes sense to be per-service, +like `service_type` or `endpoint` or `api_version` can be set by prefixing +the setting with the default service type. That might strike you funny when +setting `service_type` and it does me too - but that's just the world we live +in. + +Auth Settings +------------- + +Keystone has auth plugins - which means it's not possible to know ahead of time +which auth settings are needed. `os-client-config` sets the default plugin type +to `password`, which is what things all were before plugins came about. In +order to facilitate validation of values, all of the parameters that exist +as a result of a chosen plugin need to go into the auth dict. For password +auth, this includes `auth_url`, `username` and `password` as well as anything +related to domains, projects and trusts. + +Splitting Secrets +----------------- + +In some scenarios, such as configuration management controlled environments, +it might be easier to have secrets in one file and non-secrets in another. +This is fully supported via an optional file `secure.yaml` which follows all +the same location rules as `clouds.yaml`. It can contain anything you put +in `clouds.yaml` and will take precedence over anything in the `clouds.yaml` +file. + +.. code-block:: yaml + + # clouds.yaml + clouds: + internap: + profile: internap + auth: + username: api-55f9a00fb2619 + project_name: inap-17037 + regions: + - ams01 + - nyj01 + # secure.yaml + clouds: + internap: + auth: + password: XXXXXXXXXXXXXXXXX + +SSL Settings +------------ + +When the access to a cloud is done via a secure connection, `os-client-config` +will always verify the SSL cert by default. This can be disabled by setting +`verify` to `False`. In case the cert is signed by an unknown CA, a specific +cacert can be provided via `cacert`. **WARNING:** `verify` will always have +precedence over `cacert`, so when setting a CA cert but disabling `verify`, the +cloud cert will never be validated. + +Client certs are also configurable. `cert` will be the client cert file +location. In case the cert key is not included within the client cert file, +its file location needs to be set via `key`. + +.. code-block:: yaml + + # clouds.yaml + clouds: + secure: + auth: ... + key: /home/myhome/client-cert.key + cert: /home/myhome/client-cert.crt + cacert: /home/myhome/ca.crt + insecure: + auth: ... + verify: False + +Cache Settings +-------------- + +Accessing a cloud is often expensive, so it's quite common to want to do some +client-side caching of those operations. To facilitate that, `os-client-config` +understands passing through cache settings to dogpile.cache, with the following +behaviors: + +* Listing no config settings means you get a null cache. +* `cache.expiration_time` and nothing else gets you memory cache. +* Otherwise, `cache.class` and `cache.arguments` are passed in + +Different cloud behaviors are also differently expensive to deal with. If you +want to get really crazy and tweak stuff, you can specify different expiration +times on a per-resource basis by passing values, in seconds to an expiration +mapping keyed on the singular name of the resource. A value of `-1` indicates +that the resource should never expire. + +`os-client-config` does not actually cache anything itself, but it collects +and presents the cache information so that your various applications that +are connecting to OpenStack can share a cache should you desire. + +.. code-block:: yaml + + cache: + class: dogpile.cache.pylibmc + expiration_time: 3600 + arguments: + url: + - 127.0.0.1 + expiration: + server: 5 + flavor: -1 + clouds: + mtvexx: + profile: vexxhost + auth: + username: mordred@inaugust.com + password: XXXXXXXXX + project_name: mordred@inaugust.com + region_name: ca-ymq-1 + dns_api_version: 1 + + +IPv6 +---- + +IPv6 is the future, and you should always use it if your cloud supports it and +if your local network supports it. Both of those are easily detectable and all +friendly software should do the right thing. However, sometimes you might +exist in a location where you have an IPv6 stack, but something evil has +caused it to not actually function. In that case, there is a config option +you can set to unbreak you `force_ipv4`, or `OS_FORCE_IPV4` boolean +environment variable. + +.. code-block:: yaml + + client: + force_ipv4: true + clouds: + mtvexx: + profile: vexxhost + auth: + username: mordred@inaugust.com + password: XXXXXXXXX + project_name: mordred@inaugust.com + region_name: ca-ymq-1 + dns_api_version: 1 + monty: + profile: rax + auth: + username: mordred@inaugust.com + password: XXXXXXXXX + project_name: mordred@inaugust.com + region_name: DFW + +The above snippet will tell client programs to prefer returning an IPv4 +address. + +Per-region settings +------------------- + +Sometimes you have a cloud provider that has config that is common to the +cloud, but also with some things you might want to express on a per-region +basis. For instance, Internap provides a public and private network specific +to the user in each region, and putting the values of those networks into +config can make consuming programs more efficient. + +To support this, the region list can actually be a list of dicts, and any +setting that can be set at the cloud level can be overridden for that +region. + +.. code-block:: yaml + + clouds: + internap: + profile: internap + auth: + password: XXXXXXXXXXXXXXXXX + username: api-55f9a00fb2619 + project_name: inap-17037 + regions: + - name: ams01 + values: + networks: + - name: inap-17037-WAN1654 + routes_externally: true + - name: inap-17037-LAN6745 + - name: nyj01 + values: + networks: + - name: inap-17037-WAN1654 + routes_externally: true + - name: inap-17037-LAN6745 diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 000000000..ec31c102c --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,12 @@ +======================== + Using os-client-config +======================== + +.. toctree:: + :maxdepth: 2 + + configuration + using + vendor-support + network-config + releasenotes diff --git a/doc/source/network-config.rst b/doc/source/user/network-config.rst similarity index 100% rename from doc/source/network-config.rst rename to doc/source/user/network-config.rst diff --git a/doc/source/releasenotes.rst b/doc/source/user/releasenotes.rst similarity index 100% rename from doc/source/releasenotes.rst rename to doc/source/user/releasenotes.rst diff --git a/doc/source/user/using.rst b/doc/source/user/using.rst new file mode 100644 index 000000000..7d1d34eae --- /dev/null +++ b/doc/source/user/using.rst @@ -0,0 +1,182 @@ +========================================== + Using os-client-config in an Application +========================================== + +Usage +----- + +The simplest and least useful thing you can do is: + +.. code-block:: python + + python -m os_client_config.config + +Which will print out whatever if finds for your config. If you want to use +it from python, which is much more likely what you want to do, things like: + +Get a named cloud. + +.. code-block:: python + + import os_client_config + + cloud_config = os_client_config.OpenStackConfig().get_one_cloud( + 'internap', region_name='ams01') + print(cloud_config.name, cloud_config.region, cloud_config.config) + +Or, get all of the clouds. + +.. code-block:: python + + import os_client_config + + cloud_config = os_client_config.OpenStackConfig().get_all_clouds() + for cloud in cloud_config: + print(cloud.name, cloud.region, cloud.config) + +argparse +-------- + +If you're using os-client-config from a program that wants to process +command line options, there is a registration function to register the +arguments that both os-client-config and keystoneauth know how to deal +with - as well as a consumption argument. + +.. code-block:: python + + import argparse + import sys + + import os_client_config + + cloud_config = os_client_config.OpenStackConfig() + parser = argparse.ArgumentParser() + cloud_config.register_argparse_arguments(parser, sys.argv) + + options = parser.parse_args() + + cloud = cloud_config.get_one_cloud(argparse=options) + +Constructing OpenStack SDK object +--------------------------------- + +If what you want to do is get an OpenStack SDK Connection and you want it to +do all the normal things related to clouds.yaml, `OS_` environment variables, +a helper function is provided. The following will get you a fully configured +`openstacksdk` instance. + +.. code-block:: python + + import os_client_config + + sdk = os_client_config.make_sdk() + +If you want to do the same thing but on a named cloud. + +.. code-block:: python + + import os_client_config + + sdk = os_client_config.make_sdk(cloud='mtvexx') + +If you want to do the same thing but also support command line parsing. + +.. code-block:: python + + import argparse + + import os_client_config + + sdk = os_client_config.make_sdk(options=argparse.ArgumentParser()) + +It should be noted that OpenStack SDK has ways to construct itself that allow +for additional flexibility. If the helper function here does not meet your +needs, you should see the `from_config` method of +`openstack.connection.Connection `_ + +Constructing shade objects +-------------------------- + +If what you want to do is get a +`shade `_ OpenStackCloud object, a +helper function that honors clouds.yaml and `OS_` environment variables is +provided. The following will get you a fully configured `OpenStackCloud` +instance. + +.. code-block:: python + + import os_client_config + + cloud = os_client_config.make_shade() + +If you want to do the same thing but on a named cloud. + +.. code-block:: python + + import os_client_config + + cloud = os_client_config.make_shade(cloud='mtvexx') + +If you want to do the same thing but also support command line parsing. + +.. code-block:: python + + import argparse + + import os_client_config + + cloud = os_client_config.make_shade(options=argparse.ArgumentParser()) + +Constructing REST API Clients +----------------------------- + +What if you want to make direct REST calls via a Session interface? You're +in luck. A similar interface is available as with `openstacksdk` and `shade`. +The main difference is that you need to specify which service you want to +talk to and `make_rest_client` will return you a keystoneauth Session object +that is mounted on the endpoint for the service you're looking for. + +.. code-block:: python + + import os_client_config + + session = os_client_config.make_rest_client('compute', cloud='vexxhost') + + response = session.get('/servers') + server_list = response.json()['servers'] + +Constructing Legacy Client objects +---------------------------------- + +If you want get an old-style Client object from a python-\*client library, +and you want it to do all the normal things related to clouds.yaml, `OS_` +environment variables, a helper function is also provided. The following +will get you a fully configured `novaclient` instance. + +.. code-block:: python + + import os_client_config + + nova = os_client_config.make_client('compute') + +If you want to do the same thing but on a named cloud. + +.. code-block:: python + + import os_client_config + + nova = os_client_config.make_client('compute', cloud='mtvexx') + +If you want to do the same thing but also support command line parsing. + +.. code-block:: python + + import argparse + + import os_client_config + + nova = os_client_config.make_client( + 'compute', options=argparse.ArgumentParser()) + +If you want to get fancier than that in your python, then the rest of the +API is available to you. But often times, you just want to do the one thing. diff --git a/doc/source/vendor-support.rst b/doc/source/user/vendor-support.rst similarity index 100% rename from doc/source/vendor-support.rst rename to doc/source/user/vendor-support.rst From 412f0fdd8503d4a5b67d5ec55021fd0a9f452984 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Jun 2017 15:17:54 -0400 Subject: [PATCH 349/365] turn on warning-is-error in documentation build Change-Id: I18cdecec84f8dd5f11741ac1ffc35630f7eb64b8 Signed-off-by: Doug Hellmann --- os_client_config/config.py | 1 + setup.cfg | 1 + 2 files changed, 2 insertions(+) diff --git a/os_client_config/config.py b/os_client_config/config.py index 1ed416cf5..eb415f702 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -661,6 +661,7 @@ class OpenStackConfig(object): the keystoneauth Auth Plugin Options and os-cloud. Also, peek in the argv to see if all of the auth plugin options should be registered or merely the ones already configured. + :param argparse.ArgumentParser: parser to attach argparse options to :param list argv: the arguments provided to the application :param string service_keys: Service or list of services this argparse diff --git a/setup.cfg b/setup.cfg index b87bd6ab3..19548fb75 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ packages = source-dir = doc/source build-dir = doc/build all_files = 1 +warning-is-error = 1 [upload_sphinx] upload-dir = doc/build/html From 30c8729f782c0c13ca872260085d5b7b7c37df61 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Jun 2017 15:26:16 -0400 Subject: [PATCH 350/365] switch from oslosphinx to openstackdocstheme Change-Id: Ie45909df0b5a118d0200a1ee71277f4dbfe41d08 Signed-off-by: Doug Hellmann --- doc/source/conf.py | 5 ++++- releasenotes/source/conf.py | 5 +++-- test-requirements.txt | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 208517c86..82f27c36a 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -15,6 +15,8 @@ import os import sys +import openstackdocstheme + sys.path.insert(0, os.path.abspath('../..')) # -- General configuration ---------------------------------------------------- @@ -23,7 +25,6 @@ sys.path.insert(0, os.path.abspath('../..')) extensions = [ 'sphinx.ext.autodoc', #'sphinx.ext.intersphinx', - 'oslosphinx', 'reno.sphinxext' ] @@ -58,6 +59,8 @@ pygments_style = 'sphinx' # html_theme_path = ["."] # html_theme = '_theme' # html_static_path = ['static'] +html_theme = 'openstackdocs' +html_theme_path = [openstackdocstheme.get_html_theme_path()] # Output file base name for HTML help builder. htmlhelp_basename = '%sdoc' % project diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index e33ee8e57..1d0365645 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -21,12 +21,12 @@ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' +import openstackdocstheme # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'oslosphinx', 'reno.sphinxext', ] @@ -100,7 +100,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -109,6 +109,7 @@ html_theme = 'default' # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] +html_theme_path = [openstackdocstheme.get_html_theme_path()] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". diff --git a/test-requirements.txt b/test-requirements.txt index e71d004cd..5fc33b015 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,7 +13,7 @@ mock>=1.2 python-glanceclient>=0.18.0 python-subunit>=0.0.18 sphinx>=1.5.1 # BSD -oslosphinx>=4.7.0 # Apache-2.0 +openstackdocstheme>=1.5.0 # Apache-2.0 oslotest>=1.5.1,<1.6.0 # Apache-2.0 reno>=0.1.1 # Apache2 testrepository>=0.0.18 From f74902b0b9f82e221b1ae1af7f254dc2918f96b8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 23 Jun 2017 14:53:54 -0400 Subject: [PATCH 351/365] use openstackdocstheme html context Set some of the new config values and enable openstackdocstheme as an extension so it will inject values into the page context as it writes each documentation page. This ensures the pages link to the right bug tracker, etc. Change-Id: Id9cc61e81aa43f4b69883d338090716005477d0a Signed-off-by: Doug Hellmann --- doc/source/conf.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 82f27c36a..cbd9888d1 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -25,9 +25,16 @@ sys.path.insert(0, os.path.abspath('../..')) extensions = [ 'sphinx.ext.autodoc', #'sphinx.ext.intersphinx', - 'reno.sphinxext' + 'reno.sphinxext', + 'openstackdocstheme', ] +# openstackdocstheme options +repository_name = 'openstack/os-client-config' +bug_project = 'os-client-config' +bug_tag = '' +html_last_updated_fmt = '%Y-%m-%d %H:%M' + # autodoc generation is a bit aggressive and a nuisance when doing heavy # text edit cycles. # execute "export SPHINX_DEBUG=1" in your terminal to disable From 623593a6933da69000967abd7b6d3e2cf7cec954 Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Sat, 15 Jul 2017 20:26:43 +0200 Subject: [PATCH 352/365] Manually sync with g-r Change-Id: I4298bb7c2d66632b716b0dbeae64c9dca2b3434d --- requirements.txt | 8 ++++---- setup.py | 11 +++++++++-- test-requirements.txt | 30 +++++++++++++++--------------- 3 files changed, 28 insertions(+), 21 deletions(-) mode change 100755 => 100644 setup.py diff --git a/requirements.txt b/requirements.txt index 1531be808..6d1ee01aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -PyYAML>=3.1.0 -appdirs>=1.3.0 -keystoneauth1>=2.1.0 -requestsexceptions>=1.1.1 # Apache-2.0 +PyYAML>=3.10.0 # MIT +appdirs>=1.3.0 # MIT License +keystoneauth1>=3.0.1 # Apache-2.0 +requestsexceptions>=1.2.0 # Apache-2.0 diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 70c2b3f32..566d84432 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,6 +16,14 @@ # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + setuptools.setup( - setup_requires=['pbr'], + setup_requires=['pbr>=2.0.0'], pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt index 5fc33b015..9739a1696 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,20 +2,20 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0 +hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 -coverage>=3.6 +coverage!=4.4,>=4.0 # Apache-2.0 docutils>=0.11 # OSI-Approved Open Source, Public Domain -extras -fixtures>=0.3.14 -jsonschema>=2.0.0,<3.0.0,!=2.5.0 -mock>=1.2 -python-glanceclient>=0.18.0 -python-subunit>=0.0.18 -sphinx>=1.5.1 # BSD -openstackdocstheme>=1.5.0 # Apache-2.0 -oslotest>=1.5.1,<1.6.0 # Apache-2.0 -reno>=0.1.1 # Apache2 -testrepository>=0.0.18 -testscenarios>=0.4 -testtools>=0.9.36,!=1.2.0 +extras # MIT +fixtures>=3.0.0 # Apache-2.0/BSD +jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT +mock>=2.0 # BSD +python-glanceclient>=2.7.0 # Apache-2.0 +python-subunit>=0.0.18 # Apache-2.0/BSD +sphinx>=1.6.2 # BSD +openstackdocstheme>=1.11.0 # Apache-2.0 +oslotest>=1.10.0 # Apache-2.0 +reno!=2.3.1,>=1.8.0 # Apache-2.0 +testrepository>=0.0.18 # Apache-2.0/BSD +testscenarios>=0.4 # Apache-2.0/BSD +testtools>=1.4.0 # MIT From 1d4c124bb1c94ce475ca494b86dd69b09cbae0b4 Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Thu, 27 Jul 2017 20:30:31 +0000 Subject: [PATCH 353/365] Updated from global requirements Change-Id: I611163aecdc4810e6fd1d7e47e60171d72db0ea3 --- requirements.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6d1ee01aa..b4387d3ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ # process, which may cause wedges in the gate later. PyYAML>=3.10.0 # MIT appdirs>=1.3.0 # MIT License -keystoneauth1>=3.0.1 # Apache-2.0 +keystoneauth1>=3.1.0 # Apache-2.0 requestsexceptions>=1.2.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 9739a1696..3c44d7f02 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -13,7 +13,7 @@ mock>=2.0 # BSD python-glanceclient>=2.7.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD sphinx>=1.6.2 # BSD -openstackdocstheme>=1.11.0 # Apache-2.0 +openstackdocstheme>=1.16.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 reno!=2.3.1,>=1.8.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD From 8c8f365a9f143f2159339584f15a48bdf185b63d Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 28 Jul 2017 21:04:06 +0000 Subject: [PATCH 354/365] Update reno for stable/pike Change-Id: Ie69ff1e5a5d3cf3a762d6915c5d596de4f919931 --- releasenotes/source/index.rst | 1 + releasenotes/source/pike.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/pike.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 22609515d..405f263a0 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -9,6 +9,7 @@ Contents :maxdepth: 2 unreleased + pike ocata newton mitaka diff --git a/releasenotes/source/pike.rst b/releasenotes/source/pike.rst new file mode 100644 index 000000000..e43bfc0ce --- /dev/null +++ b/releasenotes/source/pike.rst @@ -0,0 +1,6 @@ +=================================== + Pike Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/pike From eed1cbb8cdbb9a1214277b5d38a4b75123ec14ef Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Sat, 5 Aug 2017 18:52:47 +0200 Subject: [PATCH 355/365] Remove OSIC OSIC has been decommissioned, remove the now useless vendor data. Change-Id: I57c6043018e96c0069c7db777b9f585cb7d535e7 Related-Change: I2d1b0710e875bd1ebc305fb5b184b68bf18f2ef7 --- os_client_config/vendors/osic.json | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 os_client_config/vendors/osic.json diff --git a/os_client_config/vendors/osic.json b/os_client_config/vendors/osic.json deleted file mode 100644 index 484d7111b..000000000 --- a/os_client_config/vendors/osic.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "osic", - "profile": { - "auth": { - "auth_url": "https://cloud1.osic.org:5000" - }, - "regions": [ - "RegionOne" - ] - } -} From 240e2594b969fee06be5e5a38e414e6a12d5ba6e Mon Sep 17 00:00:00 2001 From: lingyongxu Date: Mon, 7 Aug 2017 15:10:02 +0800 Subject: [PATCH 356/365] Update the documentation link for doc migration This patch is proposed according to the Direction 10 of doc migration(https://etherpad.openstack.org/p/doc-migration-tracking). Change-Id: Ida458338d353cbd6cc0162263db25f533b0bd9fd --- HACKING.rst | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HACKING.rst b/HACKING.rst index c995c5c92..aab08e34e 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -1,4 +1,4 @@ os-client-config Style Commandments =============================================== -Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ \ No newline at end of file +Read the OpenStack Style Commandments https://docs.openstack.org/hacking/latest diff --git a/setup.cfg b/setup.cfg index 19548fb75..7149cdf8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ description-file = README.rst author = OpenStack author-email = openstack-dev@lists.openstack.org -home-page = http://docs.openstack.org/developer/os-client-config/ +home-page = https://docs.openstack.org/os-client-config/latest classifier = Environment :: OpenStack Intended Audience :: Information Technology From d597ee271e1085e10f89fa9632491b3647ed95b1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 7 Aug 2017 15:37:37 -0700 Subject: [PATCH 357/365] Update globals safely The right way to update these globals is to use a lock and ensure that nobody else is updating them at the same time. Also update a temporary dictionary before setting the global one so that nobody sees partial updates to the global one. This should help fix the thread-safety of shade (and other tooling built ontop of this library). Change-Id: Ie0e0369d98ba6a01edcbf447378a786eec3f13f9 --- os_client_config/constructors.py | 14 +++++++++++--- os_client_config/defaults.py | 21 ++++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/os_client_config/constructors.py b/os_client_config/constructors.py index e88ac92d6..579bb2d5e 100644 --- a/os_client_config/constructors.py +++ b/os_client_config/constructors.py @@ -14,15 +14,23 @@ import json import os +import threading _json_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'constructors.json') _class_mapping = None +_class_mapping_lock = threading.Lock() def get_constructor_mapping(): global _class_mapping - if not _class_mapping: + if _class_mapping is not None: + return _class_mapping.copy() + with _class_mapping_lock: + if _class_mapping is not None: + return _class_mapping.copy() + tmp_class_mapping = {} with open(_json_path, 'r') as json_file: - _class_mapping = json.load(json_file) - return _class_mapping + tmp_class_mapping.update(json.load(json_file)) + _class_mapping = tmp_class_mapping + return tmp_class_mapping.copy() diff --git a/os_client_config/defaults.py b/os_client_config/defaults.py index c10358a10..1231cce92 100644 --- a/os_client_config/defaults.py +++ b/os_client_config/defaults.py @@ -14,19 +14,30 @@ import json import os +import threading _json_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'defaults.json') _defaults = None +_defaults_lock = threading.Lock() def get_defaults(): global _defaults - if not _defaults: + if _defaults is not None: + return _defaults.copy() + with _defaults_lock: + if _defaults is not None: + # Did someone else just finish filling it? + return _defaults.copy() # Python language specific defaults # These are defaults related to use of python libraries, they are # not qualities of a cloud. - _defaults = dict( + # + # NOTE(harlowja): update a in-memory dict, before updating + # the global one so that other callers of get_defaults do not + # see the partially filled one. + tmp_defaults = dict( api_timeout=None, verify=True, cacert=None, @@ -36,6 +47,6 @@ def get_defaults(): with open(_json_path, 'r') as json_file: updates = json.load(json_file) if updates is not None: - _defaults.update(updates) - - return _defaults.copy() + tmp_defaults.update(updates) + _defaults = tmp_defaults + return tmp_defaults.copy() From 164501c71570eac15c7be55d0400c400e502bc8b Mon Sep 17 00:00:00 2001 From: Sean Handley Date: Fri, 11 Aug 2017 14:12:12 +0100 Subject: [PATCH 358/365] DataCentred supports Keystone V3 and Glance V2. Change-Id: Ia8c656e2c6b97c877f5028fef8a94a2c41909bc5 --- os_client_config/vendors/datacentred.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/os_client_config/vendors/datacentred.json b/os_client_config/vendors/datacentred.json index 2be4a5863..e67d3da72 100644 --- a/os_client_config/vendors/datacentred.json +++ b/os_client_config/vendors/datacentred.json @@ -5,7 +5,7 @@ "auth_url": "https://compute.datacentred.io:5000" }, "region-name": "sal01", - "identity_api_version": "2", - "image_api_version": "1" + "identity_api_version": "3", + "image_api_version": "2" } } From 15a83dab9a8fa68f983903a65fb656104379da5d Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Sat, 12 Aug 2017 11:50:13 +0000 Subject: [PATCH 359/365] Updated from global requirements Change-Id: Iad6eba535f48d0f09e5507db32399623f63a4f88 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3c44d7f02..cb252f268 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,7 +10,7 @@ extras # MIT fixtures>=3.0.0 # Apache-2.0/BSD jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT mock>=2.0 # BSD -python-glanceclient>=2.7.0 # Apache-2.0 +python-glanceclient>=2.8.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD sphinx>=1.6.2 # BSD openstackdocstheme>=1.16.0 # Apache-2.0 From b5af1ae9367db42a50fa926f0f0bcd3f2ea56e9a Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Fri, 18 Aug 2017 11:39:22 +0000 Subject: [PATCH 360/365] Updated from global requirements Change-Id: I805c30c0a522c03721a97118594030b9c8dfcd51 --- requirements.txt | 2 +- test-requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index b4387d3ba..8f0e21321 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -PyYAML>=3.10.0 # MIT +PyYAML>=3.10 # MIT appdirs>=1.3.0 # MIT License keystoneauth1>=3.1.0 # Apache-2.0 requestsexceptions>=1.2.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index cb252f268..9a3518a43 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -9,13 +9,13 @@ docutils>=0.11 # OSI-Approved Open Source, Public Domain extras # MIT fixtures>=3.0.0 # Apache-2.0/BSD jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT -mock>=2.0 # BSD +mock>=2.0.0 # BSD python-glanceclient>=2.8.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD sphinx>=1.6.2 # BSD openstackdocstheme>=1.16.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 -reno!=2.3.1,>=1.8.0 # Apache-2.0 +reno>=2.5.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD testtools>=1.4.0 # MIT From 371c3ebc0ee28c5e6c0d942c6efc8b16553b80db Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Fri, 1 Sep 2017 12:44:43 +0000 Subject: [PATCH 361/365] Updated from global requirements Change-Id: I7b1217cd50a3e3c89edcb5bda1d22cded90e9b4b --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8f0e21321..6c609105e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ # process, which may cause wedges in the gate later. PyYAML>=3.10 # MIT appdirs>=1.3.0 # MIT License -keystoneauth1>=3.1.0 # Apache-2.0 +keystoneauth1>=3.2.0 # Apache-2.0 requestsexceptions>=1.2.0 # Apache-2.0 From 3fb4fec1d6ab2c72f72997533792fbe5d05936de Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Wed, 13 Sep 2017 13:00:03 +0000 Subject: [PATCH 362/365] Updated from global requirements Change-Id: I6ecb4e80d2944bf592a2cbd41695643bc49f832d --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 9a3518a43..bbe53373d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,12 +8,12 @@ coverage!=4.4,>=4.0 # Apache-2.0 docutils>=0.11 # OSI-Approved Open Source, Public Domain extras # MIT fixtures>=3.0.0 # Apache-2.0/BSD -jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT +jsonschema<3.0.0,>=2.6.0 # MIT mock>=2.0.0 # BSD python-glanceclient>=2.8.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD sphinx>=1.6.2 # BSD -openstackdocstheme>=1.16.0 # Apache-2.0 +openstackdocstheme>=1.17.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 reno>=2.5.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD From 6f72637fb6356d39adee82a2718118156ae4a6bf Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Thu, 14 Sep 2017 16:57:42 -0500 Subject: [PATCH 363/365] Updates for stestr Change-Id: I344cd6ce38d8db8fe24e1611c9c61e1ffa1b586d --- .gitignore | 1 + .stestr.conf | 3 +++ tox.ini | 3 +++ 3 files changed, 7 insertions(+) create mode 100644 .stestr.conf diff --git a/.gitignore b/.gitignore index 218bf6a45..c24b89b8a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ cover .coverage .tox nosetests.xml +.stestr/ .testrepository # Translations diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 000000000..792382206 --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=. +top_dir=./ diff --git a/tox.ini b/tox.ini index bf0fd8bf9..57feb6d0b 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,9 @@ setenv = VIRTUAL_ENV={envdir} BRANCH_NAME=master CLIENT_NAME=os-client-config + OS_STDOUT_CAPTURE=1 + OS_STDERR_CAPTURE=1 + OS_TEST_TIMEOUT=60 deps = -r{toxinidir}/test-requirements.txt commands = python setup.py testr --slowest --testr-args='{posargs}' From eafc8bed564d41f2a0c6a050e7a301241309fa59 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 16 Sep 2017 13:13:25 -0500 Subject: [PATCH 364/365] Fix requires_floating_ip This isn't a required piece of the config, so it might be unset. Use get instead of []. Change-Id: I1bbbcb4ac63a4f6d4399c0fa8881c21264a03e4b --- os_client_config/cloud_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os_client_config/cloud_config.py b/os_client_config/cloud_config.py index 9c6b5a5c4..d8b1e262c 100644 --- a/os_client_config/cloud_config.py +++ b/os_client_config/cloud_config.py @@ -505,7 +505,7 @@ class CloudConfig(object): """ if self.config['floating_ip_source'] == "None": return False - return self.config['requires_floating_ip'] + return self.config.get('requires_floating_ip') def get_external_networks(self): """Get list of network names for external networks.""" From 94ace709e2bf860567a0920c3f836263432862ea Mon Sep 17 00:00:00 2001 From: OpenStack Proposal Bot Date: Sat, 16 Sep 2017 23:21:20 +0000 Subject: [PATCH 365/365] Updated from global requirements Change-Id: I1bbda934cc65d508f1cece8c5adc714e9f464707 --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index bbe53373d..73d982263 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,7 +6,7 @@ hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 docutils>=0.11 # OSI-Approved Open Source, Public Domain -extras # MIT +extras>=0.0.3 # MIT fixtures>=3.0.0 # Apache-2.0/BSD jsonschema<3.0.0,>=2.6.0 # MIT mock>=2.0.0 # BSD