From f4aeb75acf8ea3526f70af2ea2ac64ac05f9892d Mon Sep 17 00:00:00 2001 From: Henry Nash Date: Thu, 8 Nov 2012 20:15:07 +0000 Subject: [PATCH 001/120] Make initial structural changes to keystoneclient in preparation to moving auth_token here from keystone. No functional change should occur from this commit (even though it did refresh a newer copy of openstack.common.setup.py, none of the newer updates are in functions called from this client) blueprint authtoken-to-keystoneclient-repo Change-Id: Ie54feb73e0e34b56400fdaa8a8f876f9284bac27 --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b From 8f7d90fc1ab1ad4a5ffd03a23deae0ff8dd314f1 Mon Sep 17 00:00:00 2001 From: Henry Nash Date: Mon, 12 Nov 2012 19:40:21 +0000 Subject: [PATCH 002/120] Add auth-token code to keystoneclient, along with supporting files This step in the process duplicates the auth-token code to keystoneclient but, for the moment, leaves a copy in its origional location in keystone. Testing for auth-token is also copied across, as is the cms support file. Although no other project will yet pick up the code here in the client, since the paste.ini files haev not yet been updated, it would work if anyone did reference it. Once the client code is in, the next step is to update all the other project paste files, and then finally retire the code from keystone. Change-Id: I88853a373d406020d54b61cba5a5e887380e3b3e --- auth_token.py | 854 ++++++++++++++++++++++++++++++++++++++++++++++++++ test.py | 67 ++++ 2 files changed, 921 insertions(+) create mode 100644 auth_token.py create mode 100644 test.py diff --git a/auth_token.py b/auth_token.py new file mode 100644 index 00000000..93af6e14 --- /dev/null +++ b/auth_token.py @@ -0,0 +1,854 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010-2012 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +TOKEN-BASED AUTH MIDDLEWARE + +This WSGI component: + +* Verifies that incoming client requests have valid tokens by validating + tokens with the auth service. +* Rejects unauthenticated requests UNLESS it is in 'delay_auth_decision' + mode, which means the final decision is delegated to the downstream WSGI + component (usually the OpenStack service) +* Collects and forwards identity information based on a valid token + such as user name, tenant, etc + +Refer to: http://keystone.openstack.org/middlewarearchitecture.html + +HEADERS +------- + +* Headers starting with HTTP\_ is a standard http header +* Headers starting with HTTP_X is an extended http header + +Coming in from initial call from client or customer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +HTTP_X_AUTH_TOKEN + The client token being passed in. + +HTTP_X_STORAGE_TOKEN + The client token being passed in (legacy Rackspace use) to support + swift/cloud files + +Used for communication between components +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +WWW-Authenticate + HTTP header returned to a user indicating which endpoint to use + to retrieve a new token + +What we add to the request for use by the OpenStack service +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +HTTP_X_IDENTITY_STATUS + 'Confirmed' or 'Invalid' + The underlying service will only see a value of 'Invalid' if the Middleware + is configured to run in 'delay_auth_decision' mode + +HTTP_X_TENANT_ID + Identity service managed unique identifier, string + +HTTP_X_TENANT_NAME + Unique tenant identifier, string + +HTTP_X_USER_ID + Identity-service managed unique identifier, string + +HTTP_X_USER_NAME + Unique user identifier, string + +HTTP_X_ROLES + Comma delimited list of case-sensitive Roles + +HTTP_X_SERVICE_CATALOG + json encoded keystone service catalog (optional). + +HTTP_X_TENANT + *Deprecated* in favor of HTTP_X_TENANT_ID and HTTP_X_TENANT_NAME + Keystone-assigned unique identifier, deprecated + +HTTP_X_USER + *Deprecated* in favor of HTTP_X_USER_ID and HTTP_X_USER_NAME + Unique user name, string + +HTTP_X_ROLE + *Deprecated* in favor of HTTP_X_ROLES + This is being renamed, and the new header contains the same data. + +""" + +import datetime +import httplib +import json +import logging +import os +import stat +import time +import webob +import webob.exc + +from keystoneclient.openstack.common import jsonutils +from keystoneclient.common import cms +from keystoneclient import utils +from keystoneclient.openstack.common import timeutils + +CONF = None +try: + from openstack.common import cfg + CONF = cfg.CONF +except ImportError: + # cfg is not a library yet, try application copies + for app in 'nova', 'glance', 'quantum', 'cinder': + try: + cfg = __import__('%s.openstack.common.cfg' % app, + fromlist=['%s.openstack.common' % app]) + # test which application middleware is running in + if hasattr(cfg, 'CONF') and 'config_file' in cfg.CONF: + CONF = cfg.CONF + break + except ImportError: + pass +if not CONF: + from keystoneclient.openstack.common import cfg + CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +# alternative middleware configuration in the main application's +# configuration file e.g. in nova.conf +# [keystone_authtoken] +# auth_host = 127.0.0.1 +# auth_port = 35357 +# auth_protocol = http +# admin_tenant_name = admin +# admin_user = admin +# admin_password = badpassword +opts = [ + cfg.StrOpt('auth_admin_prefix', default=''), + cfg.StrOpt('auth_host', default='127.0.0.1'), + cfg.IntOpt('auth_port', default=35357), + cfg.StrOpt('auth_protocol', default='https'), + cfg.StrOpt('auth_uri', default=None), + cfg.BoolOpt('delay_auth_decision', default=False), + cfg.StrOpt('admin_token'), + cfg.StrOpt('admin_user'), + cfg.StrOpt('admin_password'), + cfg.StrOpt('admin_tenant_name', default='admin'), + cfg.StrOpt('certfile'), + cfg.StrOpt('keyfile'), + cfg.StrOpt('signing_dir'), + cfg.ListOpt('memcache_servers'), + cfg.IntOpt('token_cache_time', default=300), +] +CONF.register_opts(opts, group='keystone_authtoken') + + +def will_expire_soon(expiry): + """ Determines if expiration is about to occur. + + :param expiry: a datetime of the expected expiration + :returns: boolean : true if expiration is within 30 seconds + """ + soon = (timeutils.utcnow() + datetime.timedelta(seconds=30)) + return expiry < soon + + +class InvalidUserToken(Exception): + pass + + +class ServiceError(Exception): + pass + + +class ConfigurationError(Exception): + pass + + +class AuthProtocol(object): + """Auth Middleware that handles authenticating client calls.""" + + def __init__(self, app, conf): + LOG.info('Starting keystone auth_token middleware') + self.conf = conf + self.app = app + + # delay_auth_decision means we still allow unauthenticated requests + # through and we let the downstream service make the final decision + self.delay_auth_decision = (self._conf_get('delay_auth_decision') in + (True, 'true', 't', '1', 'on', 'yes', 'y')) + + # where to find the auth service (we use this to validate tokens) + self.auth_host = self._conf_get('auth_host') + self.auth_port = int(self._conf_get('auth_port')) + self.auth_protocol = self._conf_get('auth_protocol') + if self.auth_protocol == 'http': + self.http_client_class = httplib.HTTPConnection + else: + self.http_client_class = httplib.HTTPSConnection + + self.auth_admin_prefix = self._conf_get('auth_admin_prefix') + self.auth_uri = self._conf_get('auth_uri') + if self.auth_uri is None: + self.auth_uri = '%s://%s:%s' % (self.auth_protocol, + self.auth_host, + self.auth_port) + + # SSL + self.cert_file = self._conf_get('certfile') + self.key_file = self._conf_get('keyfile') + + #signing + self.signing_dirname = self._conf_get('signing_dir') + if self.signing_dirname is None: + self.signing_dirname = '%s/keystone-signing' % os.environ['HOME'] + LOG.info('Using %s as cache directory for signing certificate' % + self.signing_dirname) + if (os.path.exists(self.signing_dirname) and + not os.access(self.signing_dirname, os.W_OK)): + raise ConfigurationError("unable to access signing dir %s" % + self.signing_dirname) + + if not os.path.exists(self.signing_dirname): + os.makedirs(self.signing_dirname) + #will throw IOError if it cannot change permissions + os.chmod(self.signing_dirname, stat.S_IRWXU) + + val = '%s/signing_cert.pem' % self.signing_dirname + self.signing_cert_file_name = val + val = '%s/cacert.pem' % self.signing_dirname + self.ca_file_name = val + val = '%s/revoked.pem' % self.signing_dirname + self.revoked_file_name = val + + # Credentials used to verify this component with the Auth service since + # validating tokens is a privileged call + self.admin_token = self._conf_get('admin_token') + self.admin_token_expiry = None + self.admin_user = self._conf_get('admin_user') + self.admin_password = self._conf_get('admin_password') + self.admin_tenant_name = self._conf_get('admin_tenant_name') + + # Token caching via memcache + self._cache = None + self._iso8601 = None + memcache_servers = self._conf_get('memcache_servers') + # By default the token will be cached for 5 minutes + self.token_cache_time = int(self._conf_get('token_cache_time')) + self._token_revocation_list = None + self._token_revocation_list_fetched_time = None + cache_timeout = datetime.timedelta(seconds=0) + self.token_revocation_list_cache_timeout = cache_timeout + if memcache_servers: + try: + import memcache + import iso8601 + LOG.info('Using memcache for caching token') + self._cache = memcache.Client(memcache_servers.split(',')) + self._iso8601 = iso8601 + except ImportError as e: + LOG.warn('disabled caching due to missing libraries %s', e) + + def _conf_get(self, name): + # try config from paste-deploy first + if name in self.conf: + return self.conf[name] + else: + return CONF.keystone_authtoken[name] + + def __call__(self, env, start_response): + """Handle incoming request. + + Authenticate send downstream on success. Reject request if + we can't authenticate. + + """ + LOG.debug('Authenticating user token') + try: + self._remove_auth_headers(env) + user_token = self._get_user_token_from_header(env) + token_info = self._validate_user_token(user_token) + user_headers = self._build_user_headers(token_info) + self._add_headers(env, user_headers) + return self.app(env, start_response) + + except InvalidUserToken: + if self.delay_auth_decision: + LOG.info('Invalid user token - deferring reject downstream') + self._add_headers(env, {'X-Identity-Status': 'Invalid'}) + return self.app(env, start_response) + else: + LOG.info('Invalid user token - rejecting request') + return self._reject_request(env, start_response) + + except ServiceError as e: + LOG.critical('Unable to obtain admin token: %s' % e) + resp = webob.exc.HTTPServiceUnavailable() + return resp(env, start_response) + + def _remove_auth_headers(self, env): + """Remove headers so a user can't fake authentication. + + :param env: wsgi request environment + + """ + auth_headers = ( + 'X-Identity-Status', + 'X-Tenant-Id', + 'X-Tenant-Name', + 'X-User-Id', + 'X-User-Name', + 'X-Roles', + 'X-Service-Catalog', + # Deprecated + 'X-User', + 'X-Tenant', + 'X-Role', + ) + LOG.debug('Removing headers from request environment: %s' % + ','.join(auth_headers)) + self._remove_headers(env, auth_headers) + + def _get_user_token_from_header(self, env): + """Get token id from request. + + :param env: wsgi request environment + :return token id + :raises InvalidUserToken if no token is provided in request + + """ + token = self._get_header(env, 'X-Auth-Token', + self._get_header(env, 'X-Storage-Token')) + if token: + return token + else: + LOG.warn("Unable to find authentication token in headers: %s", env) + raise InvalidUserToken('Unable to find token in headers') + + def _reject_request(self, env, start_response): + """Redirect client to auth server. + + :param env: wsgi request environment + :param start_response: wsgi response callback + :returns HTTPUnauthorized http response + + """ + headers = [('WWW-Authenticate', 'Keystone uri=\'%s\'' % self.auth_uri)] + resp = webob.exc.HTTPUnauthorized('Authentication required', headers) + return resp(env, start_response) + + def get_admin_token(self): + """Return admin token, possibly fetching a new one. + + if self.admin_token_expiry is set from fetching an admin token, check + it for expiration, and request a new token is the existing token + is about to expire. + + :return admin token id + :raise ServiceError when unable to retrieve token from keystone + + """ + if self.admin_token_expiry: + if will_expire_soon(self.admin_token_expiry): + self.admin_token = None + + if not self.admin_token: + (self.admin_token, + self.admin_token_expiry) = self._request_admin_token() + + return self.admin_token + + def _get_http_connection(self): + if self.auth_protocol == 'http': + return self.http_client_class(self.auth_host, self.auth_port) + else: + return self.http_client_class(self.auth_host, + self.auth_port, + self.key_file, + self.cert_file) + + def _http_request(self, method, path): + """HTTP request helper used to make unspecified content type requests. + + :param method: http method + :param path: relative request url + :return (http response object) + :raise ServerError when unable to communicate with keystone + + """ + conn = self._get_http_connection() + + try: + conn.request(method, path) + response = conn.getresponse() + body = response.read() + except Exception as e: + LOG.error('HTTP connection exception: %s' % e) + raise ServiceError('Unable to communicate with keystone') + finally: + conn.close() + + return response, body + + def _json_request(self, method, path, body=None, additional_headers=None): + """HTTP request helper used to make json requests. + + :param method: http method + :param path: relative request url + :param body: dict to encode to json as request body. Optional. + :param additional_headers: dict of additional headers to send with + http request. Optional. + :return (http response object, response body parsed as json) + :raise ServerError when unable to communicate with keystone + + """ + conn = self._get_http_connection() + + kwargs = { + 'headers': { + 'Content-type': 'application/json', + 'Accept': 'application/json', + }, + } + + if additional_headers: + kwargs['headers'].update(additional_headers) + + if body: + kwargs['body'] = jsonutils.dumps(body) + + full_path = self.auth_admin_prefix + path + try: + conn.request(method, full_path, **kwargs) + response = conn.getresponse() + body = response.read() + except Exception as e: + LOG.error('HTTP connection exception: %s' % e) + raise ServiceError('Unable to communicate with keystone') + finally: + conn.close() + + try: + data = jsonutils.loads(body) + except ValueError: + LOG.debug('Keystone did not return json-encoded body') + data = {} + + return response, data + + def _request_admin_token(self): + """Retrieve new token as admin user from keystone. + + :return token id upon success + :raises ServerError when unable to communicate with keystone + + """ + params = { + 'auth': { + 'passwordCredentials': { + 'username': self.admin_user, + 'password': self.admin_password, + }, + 'tenantName': self.admin_tenant_name, + } + } + + response, data = self._json_request('POST', + '/v2.0/tokens', + body=params) + + try: + token = data['access']['token']['id'] + expiry = data['access']['token']['expires'] + assert token + assert expiry + datetime_expiry = timeutils.parse_isotime(expiry) + return (token, timeutils.normalize_time(datetime_expiry)) + except (AssertionError, KeyError): + LOG.warn("Unexpected response from keystone service: %s", data) + raise ServiceError('invalid json response') + except (ValueError): + LOG.warn("Unable to parse expiration time from token: %s", data) + raise ServiceError('invalid json response') + + def _validate_user_token(self, user_token, retry=True): + """Authenticate user using PKI + + :param user_token: user's token id + :param retry: Ignored, as it is not longer relevant + :return uncrypted body of the token if the token is valid + :raise InvalidUserToken if token is rejected + :no longer raises ServiceError since it no longer makes RPC + + """ + try: + token_id = cms.cms_hash_token(user_token) + cached = self._cache_get(token_id) + if cached: + return cached + if cms.is_ans1_token(user_token): + verified = self.verify_signed_token(user_token) + data = json.loads(verified) + else: + data = self.verify_uuid_token(user_token, retry) + self._cache_put(token_id, data) + return data + except Exception as e: + LOG.debug('Token validation failure.', exc_info=True) + self._cache_store_invalid(user_token) + LOG.warn("Authorization failed for token %s", user_token) + raise InvalidUserToken('Token authorization failed') + + def _build_user_headers(self, token_info): + """Convert token object into headers. + + Build headers that represent authenticated user: + * X_IDENTITY_STATUS: Confirmed or Invalid + * X_TENANT_ID: id of tenant if tenant is present + * X_TENANT_NAME: name of tenant if tenant is present + * X_USER_ID: id of user + * X_USER_NAME: name of user + * X_ROLES: list of roles + * X_SERVICE_CATALOG: service catalog + + Additional (deprecated) headers include: + * X_USER: name of user + * X_TENANT: For legacy compatibility before we had ID and Name + * X_ROLE: list of roles + + :param token_info: token object returned by keystone on authentication + :raise InvalidUserToken when unable to parse token object + + """ + user = token_info['access']['user'] + token = token_info['access']['token'] + roles = ','.join([role['name'] for role in user.get('roles', [])]) + + def get_tenant_info(): + """Returns a (tenant_id, tenant_name) tuple from context.""" + def essex(): + """Essex puts the tenant ID and name on the token.""" + return (token['tenant']['id'], token['tenant']['name']) + + def pre_diablo(): + """Pre-diablo, Keystone only provided tenantId.""" + return (token['tenantId'], token['tenantId']) + + def default_tenant(): + """Assume the user's default tenant.""" + return (user['tenantId'], user['tenantName']) + + for method in [essex, pre_diablo, default_tenant]: + try: + return method() + except KeyError: + pass + + raise InvalidUserToken('Unable to determine tenancy.') + + tenant_id, tenant_name = get_tenant_info() + + user_id = user['id'] + user_name = user['name'] + + rval = { + 'X-Identity-Status': 'Confirmed', + 'X-Tenant-Id': tenant_id, + 'X-Tenant-Name': tenant_name, + 'X-User-Id': user_id, + 'X-User-Name': user_name, + 'X-Roles': roles, + # Deprecated + 'X-User': user_name, + 'X-Tenant': tenant_name, + 'X-Role': roles, + } + + try: + catalog = token_info['access']['serviceCatalog'] + rval['X-Service-Catalog'] = jsonutils.dumps(catalog) + except KeyError: + pass + + return rval + + def _header_to_env_var(self, key): + """Convert header to wsgi env variable. + + :param key: http header name (ex. 'X-Auth-Token') + :return wsgi env variable name (ex. 'HTTP_X_AUTH_TOKEN') + + """ + return 'HTTP_%s' % key.replace('-', '_').upper() + + def _add_headers(self, env, headers): + """Add http headers to environment.""" + for (k, v) in headers.iteritems(): + env_key = self._header_to_env_var(k) + env[env_key] = v + + def _remove_headers(self, env, keys): + """Remove http headers from environment.""" + for k in keys: + env_key = self._header_to_env_var(k) + try: + del env[env_key] + except KeyError: + pass + + def _get_header(self, env, key, default=None): + """Get http header from environment.""" + env_key = self._header_to_env_var(key) + return env.get(env_key, default) + + def _cache_get(self, token): + """Return token information from cache. + + If token is invalid raise InvalidUserToken + return token only if fresh (not expired). + """ + if self._cache and token: + key = 'tokens/%s' % token + cached = self._cache.get(key) + if cached == 'invalid': + LOG.debug('Cached Token %s is marked unauthorized', token) + raise InvalidUserToken('Token authorization failed') + if cached: + data, expires = cached + if time.time() < float(expires): + LOG.debug('Returning cached token %s', token) + return data + else: + LOG.debug('Cached Token %s seems expired', token) + + def _cache_put(self, token, data): + """Put token data into the cache. + + Stores the parsed expire date in cache allowing + quick check of token freshness on retrieval. + """ + if self._cache and data: + key = 'tokens/%s' % token + if 'token' in data.get('access', {}): + timestamp = data['access']['token']['expires'] + expires = self._iso8601.parse_date(timestamp).strftime('%s') + else: + LOG.error('invalid token format') + return + LOG.debug('Storing %s token in memcache', token) + self._cache.set(key, + (data, expires), + time=self.token_cache_time) + + def _cache_store_invalid(self, token): + """Store invalid token in cache.""" + if self._cache: + key = 'tokens/%s' % token + LOG.debug('Marking token %s as unauthorized in memcache', token) + self._cache.set(key, + 'invalid', + time=self.token_cache_time) + + def cert_file_missing(self, called_proc_err, file_name): + return (called_proc_err.output.find(file_name) + and not os.path.exists(file_name)) + + def verify_uuid_token(self, user_token, retry=True): + """Authenticate user token with keystone. + + :param user_token: user's token id + :param retry: flag that forces the middleware to retry + user authentication when an indeterminate + response is received. Optional. + :return token object received from keystone on success + :raise InvalidUserToken if token is rejected + :raise ServiceError if unable to authenticate token + + """ + + headers = {'X-Auth-Token': self.get_admin_token()} + response, data = self._json_request('GET', + '/v2.0/tokens/%s' % user_token, + additional_headers=headers) + + if response.status == 200: + self._cache_put(user_token, data) + return data + if response.status == 404: + # FIXME(ja): I'm assuming the 404 status means that user_token is + # invalid - not that the admin_token is invalid + self._cache_store_invalid(user_token) + LOG.warn("Authorization failed for token %s", user_token) + raise InvalidUserToken('Token authorization failed') + if response.status == 401: + LOG.info('Keystone rejected admin token %s, resetting', headers) + self.admin_token = None + else: + LOG.error('Bad response code while validating token: %s' % + response.status) + if retry: + LOG.info('Retrying validation') + return self._validate_user_token(user_token, False) + else: + LOG.warn("Invalid user token: %s. Keystone response: %s.", + user_token, data) + + raise InvalidUserToken() + + def is_signed_token_revoked(self, signed_text): + """Indicate whether the token appears in the revocation list.""" + revocation_list = self.token_revocation_list + revoked_tokens = revocation_list.get('revoked', []) + if not revoked_tokens: + return + revoked_ids = (x['id'] for x in revoked_tokens) + token_id = utils.hash_signed_token(signed_text) + for revoked_id in revoked_ids: + if token_id == revoked_id: + LOG.debug('Token %s is marked as having been revoked', + token_id) + return True + return False + + def cms_verify(self, data): + """Verifies the signature of the provided data's IAW CMS syntax. + + If either of the certificate files are missing, fetch them and + retry. + """ + while True: + try: + output = cms.cms_verify(data, self.signing_cert_file_name, + self.ca_file_name) + except cms.subprocess.CalledProcessError as err: + if self.cert_file_missing(err, self.signing_cert_file_name): + self.fetch_signing_cert() + continue + if self.cert_file_missing(err, self.ca_file_name): + self.fetch_ca_cert() + continue + raise err + return output + + def verify_signed_token(self, signed_text): + """Check that the token is unrevoked and has a valid signature.""" + if self.is_signed_token_revoked(signed_text): + raise InvalidUserToken('Token has been revoked') + + formatted = cms.token_to_cms(signed_text) + return self.cms_verify(formatted) + + @property + def token_revocation_list_fetched_time(self): + if not self._token_revocation_list_fetched_time: + # If the fetched list has been written to disk, use its + # modification time. + if os.path.exists(self.revoked_file_name): + mtime = os.path.getmtime(self.revoked_file_name) + fetched_time = datetime.datetime.fromtimestamp(mtime) + # Otherwise the list will need to be fetched. + else: + fetched_time = datetime.datetime.min + self._token_revocation_list_fetched_time = fetched_time + return self._token_revocation_list_fetched_time + + @token_revocation_list_fetched_time.setter + def token_revocation_list_fetched_time(self, value): + self._token_revocation_list_fetched_time = value + + @property + def token_revocation_list(self): + timeout = (self.token_revocation_list_fetched_time + + self.token_revocation_list_cache_timeout) + list_is_current = timeutils.utcnow() < timeout + if list_is_current: + # Load the list from disk if required + if not self._token_revocation_list: + with open(self.revoked_file_name, 'r') as f: + self._token_revocation_list = jsonutils.loads(f.read()) + else: + self.token_revocation_list = self.fetch_revocation_list() + return self._token_revocation_list + + @token_revocation_list.setter + def token_revocation_list(self, value): + """Save a revocation list to memory and to disk. + + :param value: A json-encoded revocation list + + """ + self._token_revocation_list = jsonutils.loads(value) + self.token_revocation_list_fetched_time = timeutils.utcnow() + with open(self.revoked_file_name, 'w') as f: + f.write(value) + + def fetch_revocation_list(self, retry=True): + headers = {'X-Auth-Token': self.get_admin_token()} + response, data = self._json_request('GET', '/v2.0/tokens/revoked', + additional_headers=headers) + if response.status == 401: + if retry: + LOG.info('Keystone rejected admin token %s, resetting admin ' + 'token', headers) + self.admin_token = None + return self.fetch_revocation_list(retry=False) + if response.status != 200: + raise ServiceError('Unable to fetch token revocation list.') + if (not 'signed' in data): + raise ServiceError('Revocation list inmproperly formatted.') + return self.cms_verify(data['signed']) + + def fetch_signing_cert(self): + response, data = self._http_request('GET', + '/v2.0/certificates/signing') + try: + #todo check response + certfile = open(self.signing_cert_file_name, 'w') + certfile.write(data) + certfile.close() + except (AssertionError, KeyError): + LOG.warn("Unexpected response from keystone service: %s", data) + raise ServiceError('invalid json response') + + def fetch_ca_cert(self): + response, data = self._http_request('GET', + '/v2.0/certificates/ca') + try: + #todo check response + certfile = open(self.ca_file_name, 'w') + certfile.write(data) + certfile.close() + except (AssertionError, KeyError): + LOG.warn("Unexpected response from keystone service: %s", data) + raise ServiceError('invalid json response') + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def auth_filter(app): + return AuthProtocol(app, conf) + return auth_filter + + +def app_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + return AuthProtocol(None, conf) diff --git a/test.py b/test.py new file mode 100644 index 00000000..e5c11712 --- /dev/null +++ b/test.py @@ -0,0 +1,67 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# +# Test support for middleware authentication +# + +import os +import sys + + +ROOTDIR = os.path.dirname(os.path.abspath(os.curdir)) + + +def rootdir(*p): + return os.path.join(ROOTDIR, *p) + + +class NoModule(object): + """A mixin class to provide support for unloading/disabling modules.""" + + def __init__(self, *args, **kw): + super(NoModule, self).__init__(*args, **kw) + self._finders = [] + self._cleared_modules = {} + + def tearDown(self): + super(NoModule, self).tearDown() + for finder in self._finders: + sys.meta_path.remove(finder) + sys.modules.update(self._cleared_modules) + + def clear_module(self, module): + cleared_modules = {} + for fullname in sys.modules.keys(): + if fullname == module or fullname.startswith(module + '.'): + cleared_modules[fullname] = sys.modules.pop(fullname) + return cleared_modules + + def disable_module(self, module): + """Ensure ImportError for the specified module.""" + + # Clear 'module' references in sys.modules + self._cleared_modules.update(self.clear_module(module)) + + # Disallow further imports of 'module' + class NoModule(object): + def find_module(self, fullname, path): + if fullname == module or fullname.startswith(module + '.'): + raise ImportError + + finder = NoModule() + self._finders.append(finder) + sys.meta_path.insert(0, finder) From 94f0ec0a7a743d438741d15fa868bf12922d60e9 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 15 Nov 2012 16:03:58 -0600 Subject: [PATCH 003/120] Throw validation response into the environment Allow other middleware access to the raw response from the token validation step by adding it to the WSGI environment in the 'keystone.token_info' variable. Change-Id: I3849598d6eefd2bfcb04e27d723f08fb1935c231 --- auth_token.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/auth_token.py b/auth_token.py index 93af6e14..b6c2be0c 100644 --- a/auth_token.py +++ b/auth_token.py @@ -91,6 +91,15 @@ HTTP_X_ROLE *Deprecated* in favor of HTTP_X_ROLES This is being renamed, and the new header contains the same data. +OTHER ENVIRONMENT VARIABLES +--------------------------- + +keystone.token_info + Information about the token discovered in the process of + validation. This may include extended information returned by the + Keystone token validation call, as well as basic information about + the tenant and user. + """ import datetime @@ -283,6 +292,7 @@ class AuthProtocol(object): self._remove_auth_headers(env) user_token = self._get_user_token_from_header(env) token_info = self._validate_user_token(user_token) + env['keystone.token_info'] = token_info user_headers = self._build_user_headers(token_info) self._add_headers(env, user_headers) return self.app(env, start_response) From e1e80df2311fdd18d02899366638827f0b9fd8e0 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Mon, 3 Dec 2012 14:24:38 -0600 Subject: [PATCH 004/120] Don't try to split a list of memcache servers The memcache_servers configuration option is declared as a ListOpt, but when used in AuthProtocol.__init__(), we treat it as a string to be split on ',', which is wrong. Remove the .split(). No test is added because the memcache package may not be installed, and I cannot see an easy method to only test this clause if it is available. Fixes bug 1086125. Change-Id: Ifb0a18017d2407b3ccb2188b1a704c26997ba594 --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index b6c2be0c..0267812b 100644 --- a/auth_token.py +++ b/auth_token.py @@ -268,7 +268,7 @@ class AuthProtocol(object): import memcache import iso8601 LOG.info('Using memcache for caching token') - self._cache = memcache.Client(memcache_servers.split(',')) + self._cache = memcache.Client(memcache_servers) self._iso8601 = iso8601 except ImportError as e: LOG.warn('disabled caching due to missing libraries %s', e) From fb6da7f0b6d714c448f51837871e430655e435c8 Mon Sep 17 00:00:00 2001 From: Chuck Thier Date: Wed, 12 Dec 2012 17:11:34 -0600 Subject: [PATCH 005/120] Fix middleware logging for swift Swift sets 'log_name' in the conf, and this can be used to set the correct logger. This patch moves the log configuration to initialization so that the logger can be set appropriately and the middleware will log to the swift proxy logs. This also requires the following review in swift to work: https://review.openstack.org/#/c/17983/ Fixes bug #1089664 Change-Id: I315102ba277ac1f5ff92f4d79ae745bf53559328 --- auth_token.py | 88 ++++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/auth_token.py b/auth_token.py index 0267812b..3b63ef60 100644 --- a/auth_token.py +++ b/auth_token.py @@ -136,7 +136,6 @@ except ImportError: if not CONF: from keystoneclient.openstack.common import cfg CONF = cfg.CONF -LOG = logging.getLogger(__name__) # alternative middleware configuration in the main application's # configuration file e.g. in nova.conf @@ -193,7 +192,8 @@ class AuthProtocol(object): """Auth Middleware that handles authenticating client calls.""" def __init__(self, app, conf): - LOG.info('Starting keystone auth_token middleware') + self.LOG = logging.getLogger(conf.get('log_name', __name__)) + self.LOG.info('Starting keystone auth_token middleware') self.conf = conf self.app = app @@ -226,8 +226,8 @@ class AuthProtocol(object): self.signing_dirname = self._conf_get('signing_dir') if self.signing_dirname is None: self.signing_dirname = '%s/keystone-signing' % os.environ['HOME'] - LOG.info('Using %s as cache directory for signing certificate' % - self.signing_dirname) + self.LOG.info('Using %s as cache directory for signing certificate' % + self.signing_dirname) if (os.path.exists(self.signing_dirname) and not os.access(self.signing_dirname, os.W_OK)): raise ConfigurationError("unable to access signing dir %s" % @@ -267,11 +267,12 @@ class AuthProtocol(object): try: import memcache import iso8601 - LOG.info('Using memcache for caching token') + self.LOG.info('Using memcache for caching token') self._cache = memcache.Client(memcache_servers) self._iso8601 = iso8601 except ImportError as e: - LOG.warn('disabled caching due to missing libraries %s', e) + self.LOG.warn( + 'disabled caching due to missing libraries %s', e) def _conf_get(self, name): # try config from paste-deploy first @@ -287,7 +288,7 @@ class AuthProtocol(object): we can't authenticate. """ - LOG.debug('Authenticating user token') + self.LOG.debug('Authenticating user token') try: self._remove_auth_headers(env) user_token = self._get_user_token_from_header(env) @@ -299,15 +300,16 @@ class AuthProtocol(object): except InvalidUserToken: if self.delay_auth_decision: - LOG.info('Invalid user token - deferring reject downstream') + self.LOG.info( + 'Invalid user token - deferring reject downstream') self._add_headers(env, {'X-Identity-Status': 'Invalid'}) return self.app(env, start_response) else: - LOG.info('Invalid user token - rejecting request') + self.LOG.info('Invalid user token - rejecting request') return self._reject_request(env, start_response) except ServiceError as e: - LOG.critical('Unable to obtain admin token: %s' % e) + self.LOG.critical('Unable to obtain admin token: %s' % e) resp = webob.exc.HTTPServiceUnavailable() return resp(env, start_response) @@ -330,8 +332,8 @@ class AuthProtocol(object): 'X-Tenant', 'X-Role', ) - LOG.debug('Removing headers from request environment: %s' % - ','.join(auth_headers)) + self.LOG.debug('Removing headers from request environment: %s' % + ','.join(auth_headers)) self._remove_headers(env, auth_headers) def _get_user_token_from_header(self, env): @@ -347,7 +349,8 @@ class AuthProtocol(object): if token: return token else: - LOG.warn("Unable to find authentication token in headers: %s", env) + self.LOG.warn( + "Unable to find authentication token in headers: %s", env) raise InvalidUserToken('Unable to find token in headers') def _reject_request(self, env, start_response): @@ -408,7 +411,7 @@ class AuthProtocol(object): response = conn.getresponse() body = response.read() except Exception as e: - LOG.error('HTTP connection exception: %s' % e) + self.LOG.error('HTTP connection exception: %s' % e) raise ServiceError('Unable to communicate with keystone') finally: conn.close() @@ -448,7 +451,7 @@ class AuthProtocol(object): response = conn.getresponse() body = response.read() except Exception as e: - LOG.error('HTTP connection exception: %s' % e) + self.LOG.error('HTTP connection exception: %s' % e) raise ServiceError('Unable to communicate with keystone') finally: conn.close() @@ -456,7 +459,7 @@ class AuthProtocol(object): try: data = jsonutils.loads(body) except ValueError: - LOG.debug('Keystone did not return json-encoded body') + self.LOG.debug('Keystone did not return json-encoded body') data = {} return response, data @@ -490,10 +493,12 @@ class AuthProtocol(object): datetime_expiry = timeutils.parse_isotime(expiry) return (token, timeutils.normalize_time(datetime_expiry)) except (AssertionError, KeyError): - LOG.warn("Unexpected response from keystone service: %s", data) + self.LOG.warn( + "Unexpected response from keystone service: %s", data) raise ServiceError('invalid json response') except (ValueError): - LOG.warn("Unable to parse expiration time from token: %s", data) + self.LOG.warn( + "Unable to parse expiration time from token: %s", data) raise ServiceError('invalid json response') def _validate_user_token(self, user_token, retry=True): @@ -519,9 +524,9 @@ class AuthProtocol(object): self._cache_put(token_id, data) return data except Exception as e: - LOG.debug('Token validation failure.', exc_info=True) + self.LOG.debug('Token validation failure.', exc_info=True) self._cache_store_invalid(user_token) - LOG.warn("Authorization failed for token %s", user_token) + self.LOG.warn("Authorization failed for token %s", user_token) raise InvalidUserToken('Token authorization failed') def _build_user_headers(self, token_info): @@ -636,15 +641,15 @@ class AuthProtocol(object): key = 'tokens/%s' % token cached = self._cache.get(key) if cached == 'invalid': - LOG.debug('Cached Token %s is marked unauthorized', token) + self.LOG.debug('Cached Token %s is marked unauthorized', token) raise InvalidUserToken('Token authorization failed') if cached: data, expires = cached if time.time() < float(expires): - LOG.debug('Returning cached token %s', token) + self.LOG.debug('Returning cached token %s', token) return data else: - LOG.debug('Cached Token %s seems expired', token) + self.LOG.debug('Cached Token %s seems expired', token) def _cache_put(self, token, data): """Put token data into the cache. @@ -658,9 +663,9 @@ class AuthProtocol(object): timestamp = data['access']['token']['expires'] expires = self._iso8601.parse_date(timestamp).strftime('%s') else: - LOG.error('invalid token format') + self.LOG.error('invalid token format') return - LOG.debug('Storing %s token in memcache', token) + self.LOG.debug('Storing %s token in memcache', token) self._cache.set(key, (data, expires), time=self.token_cache_time) @@ -669,7 +674,8 @@ class AuthProtocol(object): """Store invalid token in cache.""" if self._cache: key = 'tokens/%s' % token - LOG.debug('Marking token %s as unauthorized in memcache', token) + self.LOG.debug( + 'Marking token %s as unauthorized in memcache', token) self._cache.set(key, 'invalid', time=self.token_cache_time) @@ -703,20 +709,21 @@ class AuthProtocol(object): # FIXME(ja): I'm assuming the 404 status means that user_token is # invalid - not that the admin_token is invalid self._cache_store_invalid(user_token) - LOG.warn("Authorization failed for token %s", user_token) + self.LOG.warn("Authorization failed for token %s", user_token) raise InvalidUserToken('Token authorization failed') if response.status == 401: - LOG.info('Keystone rejected admin token %s, resetting', headers) + self.LOG.info( + 'Keystone rejected admin token %s, resetting', headers) self.admin_token = None else: - LOG.error('Bad response code while validating token: %s' % - response.status) + self.LOG.error('Bad response code while validating token: %s' % + response.status) if retry: - LOG.info('Retrying validation') + self.LOG.info('Retrying validation') return self._validate_user_token(user_token, False) else: - LOG.warn("Invalid user token: %s. Keystone response: %s.", - user_token, data) + self.LOG.warn("Invalid user token: %s. Keystone response: %s.", + user_token, data) raise InvalidUserToken() @@ -730,8 +737,8 @@ class AuthProtocol(object): token_id = utils.hash_signed_token(signed_text) for revoked_id in revoked_ids: if token_id == revoked_id: - LOG.debug('Token %s is marked as having been revoked', - token_id) + self.LOG.debug('Token %s is marked as having been revoked', + token_id) return True return False @@ -813,8 +820,9 @@ class AuthProtocol(object): additional_headers=headers) if response.status == 401: if retry: - LOG.info('Keystone rejected admin token %s, resetting admin ' - 'token', headers) + self.LOG.info( + 'Keystone rejected admin token %s, resetting admin token', + headers) self.admin_token = None return self.fetch_revocation_list(retry=False) if response.status != 200: @@ -832,7 +840,8 @@ class AuthProtocol(object): certfile.write(data) certfile.close() except (AssertionError, KeyError): - LOG.warn("Unexpected response from keystone service: %s", data) + self.LOG.warn( + "Unexpected response from keystone service: %s", data) raise ServiceError('invalid json response') def fetch_ca_cert(self): @@ -844,7 +853,8 @@ class AuthProtocol(object): certfile.write(data) certfile.close() except (AssertionError, KeyError): - LOG.warn("Unexpected response from keystone service: %s", data) + self.LOG.warn( + "Unexpected response from keystone service: %s", data) raise ServiceError('invalid json response') From 3de5f0cf931b6da81c228778f20b34f93b589dbe Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Thu, 13 Dec 2012 12:31:06 -0600 Subject: [PATCH 006/120] URL-encode user-supplied tokens (bug 974319) Change-Id: I7440f879edb8d61ea2382d5d4a56e32eacce4cfd --- auth_token.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index 0267812b..b96ea006 100644 --- a/auth_token.py +++ b/auth_token.py @@ -109,6 +109,7 @@ import logging import os import stat import time +import urllib import webob import webob.exc @@ -177,6 +178,11 @@ def will_expire_soon(expiry): return expiry < soon +def safe_quote(s): + """URL-encode strings that are not already URL-encoded.""" + return urllib.quote(s) if s == urllib.unquote(s) else s + + class InvalidUserToken(Exception): pass @@ -692,9 +698,10 @@ class AuthProtocol(object): """ headers = {'X-Auth-Token': self.get_admin_token()} - response, data = self._json_request('GET', - '/v2.0/tokens/%s' % user_token, - additional_headers=headers) + response, data = self._json_request( + 'GET', + '/v2.0/tokens/%s' % safe_quote(user_token), + additional_headers=headers) if response.status == 200: self._cache_put(user_token, data) From 8decb5d95a58db6d210230cd5dbde586ea4c1d63 Mon Sep 17 00:00:00 2001 From: Guang Yee Date: Mon, 17 Dec 2012 09:05:32 -0800 Subject: [PATCH 007/120] Bug 1052674: added support for Swift cache This patch merely address the Swift cache backward compatibility support. There's another BP which address the elephant-in-the-room, which is unified and consistent caching mechanism for all middleware. See https://blueprints.launchpad.net/keystone/+spec/unifed-caching-system-for-middleware Change-Id: Iab6b4ac61c9aa9e7ad32a394557cb7558bd60a43 --- auth_token.py | 47 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/auth_token.py b/auth_token.py index 7401f827..6eb02fab 100644 --- a/auth_token.py +++ b/auth_token.py @@ -147,6 +147,13 @@ if not CONF: # admin_tenant_name = admin # admin_user = admin # admin_password = badpassword + +# when deploy Keystone auth_token middleware with Swift, user may elect +# to use Swift memcache instead of the local Keystone memcache. Swift memcache +# is passed in from the request environment and its identified by the +# 'swift.cache' key. However it could be different, depending on deployment. +# To use Swift memcache, you must set the 'cache' option to the environment +# key where the Swift cache object is stored. opts = [ cfg.StrOpt('auth_admin_prefix', default=''), cfg.StrOpt('auth_host', default='127.0.0.1'), @@ -158,6 +165,7 @@ opts = [ cfg.StrOpt('admin_user'), cfg.StrOpt('admin_password'), cfg.StrOpt('admin_tenant_name', default='admin'), + cfg.StrOpt('cache', default=None), # env key for the swift cache cfg.StrOpt('certfile'), cfg.StrOpt('keyfile'), cfg.StrOpt('signing_dir'), @@ -262,23 +270,35 @@ class AuthProtocol(object): # Token caching via memcache self._cache = None self._iso8601 = None - memcache_servers = self._conf_get('memcache_servers') + self._cache_initialized = False # cache already initialzied? # By default the token will be cached for 5 minutes self.token_cache_time = int(self._conf_get('token_cache_time')) self._token_revocation_list = None self._token_revocation_list_fetched_time = None cache_timeout = datetime.timedelta(seconds=0) self.token_revocation_list_cache_timeout = cache_timeout - if memcache_servers: - try: - import memcache - import iso8601 - self.LOG.info('Using memcache for caching token') - self._cache = memcache.Client(memcache_servers) - self._iso8601 = iso8601 - except ImportError as e: - self.LOG.warn( - 'disabled caching due to missing libraries %s', e) + + def _init_cache(self, env): + cache = self._conf_get('cache') + memcache_servers = self._conf_get('memcache_servers') + if cache and env.get(cache, None) is not None: + # use the cache from the upstream filter + self.LOG.info('Using %s memcache for caching token', cache) + self._cache = env.get(cache) + else: + # use Keystone memcache + memcache_servers = self._conf_get('memcache_servers') + if memcache_servers: + try: + import memcache + import iso8601 + self.LOG.info('Using Keystone memcache for caching token') + self._cache = memcache.Client(memcache_servers) + self._iso8601 = iso8601 + except ImportError as e: + msg = 'disabled caching due to missing libraries %s' % (e) + self.LOG.warn(msg) + self._cache_initialized = True def _conf_get(self, name): # try config from paste-deploy first @@ -295,6 +315,11 @@ class AuthProtocol(object): """ self.LOG.debug('Authenticating user token') + + # initialize memcache if we haven't done so + if not self._cache_initialized: + self._init_cache(env) + try: self._remove_auth_headers(env) user_token = self._get_user_token_from_header(env) From 2e359eedf83231dc2700b0a6f92c712b6c78c84d Mon Sep 17 00:00:00 2001 From: Adam Young Date: Wed, 19 Dec 2012 16:24:32 -0500 Subject: [PATCH 008/120] remove unused import Change-Id: I71d3da8b3fcaaf9a521c5570afe92e508bf181b7 --- auth_token.py | 1 - 1 file changed, 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 6eb02fab..dcfba1f4 100644 --- a/auth_token.py +++ b/auth_token.py @@ -110,7 +110,6 @@ import os import stat import time import urllib -import webob import webob.exc from keystoneclient.openstack.common import jsonutils From f3029a08788438bd557d8de38ea74dd5f468beeb Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Wed, 9 Jan 2013 12:59:43 -0600 Subject: [PATCH 009/120] Remove iso8601 dep in favor of openstack.common Change-Id: I83d5c8681cc3cde4889d74d50c0ffa58b96b2819 --- auth_token.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/auth_token.py b/auth_token.py index dcfba1f4..eadd2bea 100644 --- a/auth_token.py +++ b/auth_token.py @@ -268,7 +268,6 @@ class AuthProtocol(object): # Token caching via memcache self._cache = None - self._iso8601 = None self._cache_initialized = False # cache already initialzied? # By default the token will be cached for 5 minutes self.token_cache_time = int(self._conf_get('token_cache_time')) @@ -290,10 +289,8 @@ class AuthProtocol(object): if memcache_servers: try: import memcache - import iso8601 self.LOG.info('Using Keystone memcache for caching token') self._cache = memcache.Client(memcache_servers) - self._iso8601 = iso8601 except ImportError as e: msg = 'disabled caching due to missing libraries %s' % (e) self.LOG.warn(msg) @@ -691,7 +688,7 @@ class AuthProtocol(object): key = 'tokens/%s' % token if 'token' in data.get('access', {}): timestamp = data['access']['token']['expires'] - expires = self._iso8601.parse_date(timestamp).strftime('%s') + expires = timeutils.parse_isotime(timestamp).strftime('%s') else: self.LOG.error('invalid token format') return From f42fce82a74013400a9d350dd3ec27453c9db5eb Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Thu, 13 Dec 2012 12:00:59 -0600 Subject: [PATCH 010/120] Use os.path to find ~/keystone-signing (bug 1078947) Change-Id: Ie816d34299c92ba7d5cf6acf717ccfbf029f724f --- auth_token.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index dcfba1f4..33ea1153 100644 --- a/auth_token.py +++ b/auth_token.py @@ -167,7 +167,8 @@ opts = [ cfg.StrOpt('cache', default=None), # env key for the swift cache cfg.StrOpt('certfile'), cfg.StrOpt('keyfile'), - cfg.StrOpt('signing_dir'), + cfg.StrOpt('signing_dir', + default=os.path.expanduser('~/keystone-signing')), cfg.ListOpt('memcache_servers'), cfg.IntOpt('token_cache_time', default=300), ] @@ -237,8 +238,6 @@ class AuthProtocol(object): #signing self.signing_dirname = self._conf_get('signing_dir') - if self.signing_dirname is None: - self.signing_dirname = '%s/keystone-signing' % os.environ['HOME'] self.LOG.info('Using %s as cache directory for signing certificate' % self.signing_dirname) if (os.path.exists(self.signing_dirname) and From 12fdb7b8dcd8dac47ae304cb4736753319e5cde2 Mon Sep 17 00:00:00 2001 From: nachiappan-veerappan-nachiappan Date: Wed, 9 Jan 2013 15:02:32 -0800 Subject: [PATCH 011/120] Warning message is not logged for valid token-less request. if delay_auth_decision is set,auth_token middleware does not log messages for valid token-less requests. Fixes: bug #1028683 Change-Id: Ia2069686b86cc833327b11343ebaed59663fd379 --- auth_token.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 1dd72cd9..6f9d7a87 100644 --- a/auth_token.py +++ b/auth_token.py @@ -375,8 +375,10 @@ class AuthProtocol(object): if token: return token else: - self.LOG.warn( - "Unable to find authentication token in headers: %s", env) + if not self.delay_auth_decision: + self.LOG.warn("Unable to find authentication token" + " in headers") + self.LOG.debug("Headers: %s", env) raise InvalidUserToken('Unable to find token in headers') def _reject_request(self, env, start_response): From 826c291f2e69f7d14c7e1c35c6dff1fda4e5b630 Mon Sep 17 00:00:00 2001 From: Guang Yee Date: Wed, 19 Dec 2012 15:50:34 -0800 Subject: [PATCH 012/120] Blueprint memcache-protection: enable memcache value encryption/integrity check DocImpact Change-Id: I8b733256a3c2cdcf7c2ec5edac491ac4739aa847 --- auth_token.py | 105 ++++++++++++++++++++++++++++--- memcache_crypt.py | 157 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 10 deletions(-) create mode 100755 memcache_crypt.py diff --git a/auth_token.py b/auth_token.py index 6f9d7a87..7d332205 100644 --- a/auth_token.py +++ b/auth_token.py @@ -115,6 +115,7 @@ import webob.exc from keystoneclient.openstack.common import jsonutils from keystoneclient.common import cms from keystoneclient import utils +from keystoneclient.middleware import memcache_crypt from keystoneclient.openstack.common import timeutils CONF = None @@ -171,6 +172,8 @@ opts = [ default=os.path.expanduser('~/keystone-signing')), cfg.ListOpt('memcache_servers'), cfg.IntOpt('token_cache_time', default=300), + cfg.StrOpt('memcache_security_strategy', default=None), + cfg.StrOpt('memcache_secret_key', default=None), ] CONF.register_opts(opts, group='keystone_authtoken') @@ -267,7 +270,17 @@ class AuthProtocol(object): # Token caching via memcache self._cache = None + self._use_keystone_cache = False self._cache_initialized = False # cache already initialzied? + # memcache value treatment, ENCRYPT or MAC + self._memcache_security_strategy = \ + self._conf_get('memcache_security_strategy') + if self._memcache_security_strategy is not None: + self._memcache_security_strategy = \ + self._memcache_security_strategy.upper() + self._memcache_secret_key = \ + self._conf_get('memcache_secret_key') + self._assert_valid_memcache_protection_config() # By default the token will be cached for 5 minutes self.token_cache_time = int(self._conf_get('token_cache_time')) self._token_revocation_list = None @@ -275,6 +288,15 @@ class AuthProtocol(object): cache_timeout = datetime.timedelta(seconds=0) self.token_revocation_list_cache_timeout = cache_timeout + def _assert_valid_memcache_protection_config(self): + if self._memcache_security_strategy: + if self._memcache_security_strategy not in ('MAC', 'ENCRYPT'): + raise Exception('memcache_security_strategy must be ' + 'ENCRYPT or MAC') + if not self._memcache_secret_key: + raise Exception('mecmache_secret_key must be defined when ' + 'a memcache_security_strategy is defined') + def _init_cache(self, env): cache = self._conf_get('cache') memcache_servers = self._conf_get('memcache_servers') @@ -290,6 +312,7 @@ class AuthProtocol(object): import memcache self.LOG.info('Using Keystone memcache for caching token') self._cache = memcache.Client(memcache_servers) + self._use_keystone_cache = True except ImportError as e: msg = 'disabled caching due to missing libraries %s' % (e) self.LOG.warn(msg) @@ -659,6 +682,54 @@ class AuthProtocol(object): env_key = self._header_to_env_var(key) return env.get(env_key, default) + def _protect_cache_value(self, token, data): + """ Encrypt or sign data if necessary. """ + try: + if self._memcache_security_strategy == 'ENCRYPT': + return memcache_crypt.encrypt_data(token, + self._memcache_secret_key, + data) + elif self._memcache_security_strategy == 'MAC': + return memcache_crypt.sign_data(token, data) + else: + return data + except: + msg = 'Failed to encrypt/sign cache data.' + self.LOG.exception(msg) + return data + + def _unprotect_cache_value(self, token, data): + """ Decrypt or verify signed data if necessary. """ + if data is None: + return data + + try: + if self._memcache_security_strategy == 'ENCRYPT': + return memcache_crypt.decrypt_data(token, + self._memcache_secret_key, + data) + elif self._memcache_security_strategy == 'MAC': + return memcache_crypt.verify_signed_data(token, data) + else: + return data + except: + msg = 'Failed to decrypt/verify cache data.' + self.LOG.exception(msg) + # this should have the same effect as data not found in cache + return None + + def _get_cache_key(self, token): + """ Return the cache key. + + Do not use clear token as key if memcache protection is on. + + """ + htoken = token + if self._memcache_security_strategy in ('ENCRYPT', 'MAC'): + derv_token = token + self._memcache_secret_key + htoken = memcache_crypt.hash_data(derv_token) + return 'tokens/%s' % htoken + def _cache_get(self, token): """Return token information from cache. @@ -666,8 +737,9 @@ class AuthProtocol(object): return token only if fresh (not expired). """ if self._cache and token: - key = 'tokens/%s' % token + key = self._get_cache_key(token) cached = self._cache.get(key) + cached = self._unprotect_cache_value(token, cached) if cached == 'invalid': self.LOG.debug('Cached Token %s is marked unauthorized', token) raise InvalidUserToken('Token authorization failed') @@ -679,14 +751,32 @@ class AuthProtocol(object): else: self.LOG.debug('Cached Token %s seems expired', token) + def _cache_store(self, token, data, expires=None): + """ Store value into memcache. """ + key = self._get_cache_key(token) + data = self._protect_cache_value(token, data) + data_to_store = data + if expires: + data_to_store = (data, expires) + # we need to special-case set() because of the incompatibility between + # Swift MemcacheRing and python-memcached. See + # https://bugs.launchpad.net/swift/+bug/1095730 + if self._use_keystone_cache: + self._cache.set(key, + data_to_store, + time=self.token_cache_time) + else: + self._cache.set(key, + data_to_store, + timeout=self.token_cache_time) + def _cache_put(self, token, data): - """Put token data into the cache. + """ Put token data into the cache. Stores the parsed expire date in cache allowing quick check of token freshness on retrieval. """ if self._cache and data: - key = 'tokens/%s' % token if 'token' in data.get('access', {}): timestamp = data['access']['token']['expires'] expires = timeutils.parse_isotime(timestamp).strftime('%s') @@ -694,19 +784,14 @@ class AuthProtocol(object): self.LOG.error('invalid token format') return self.LOG.debug('Storing %s token in memcache', token) - self._cache.set(key, - (data, expires), - time=self.token_cache_time) + self._cache_store(token, data, expires) def _cache_store_invalid(self, token): """Store invalid token in cache.""" if self._cache: - key = 'tokens/%s' % token self.LOG.debug( 'Marking token %s as unauthorized in memcache', token) - self._cache.set(key, - 'invalid', - time=self.token_cache_time) + self._cache_store(token, 'invalid') def cert_file_missing(self, called_proc_err, file_name): return (called_proc_err.output.find(file_name) diff --git a/memcache_crypt.py b/memcache_crypt.py new file mode 100755 index 00000000..91e261da --- /dev/null +++ b/memcache_crypt.py @@ -0,0 +1,157 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010-2012 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Utilities for memcache encryption and integrity check. + +Data is serialized before been encrypted or MACed. Encryption have a +dependency on the pycrypto. If pycrypto is not available, +CryptoUnabailableError will be raised. + +Encrypted data stored in memcache are prefixed with '{ENCRYPT:AES256}'. + +MACed data stored in memcache are prefixed with '{MAC:SHA1}'. + +""" + +import base64 +import functools +import hashlib +import json +import os + +# make sure pycrypt is available +try: + from Crypto.Cipher import AES +except ImportError: + AES = None + + +# prefix marker indicating data is HMACed (signed by a secret key) +MAC_MARKER = '{MAC:SHA1}' +# prefix marker indicating data is encrypted +ENCRYPT_MARKER = '{ENCRYPT:AES256}' + + +class InvalidMacError(Exception): + """ raise when unable to verify MACed data + + This usually indicates that data had been expectedly modified in memcache. + + """ + pass + + +class DecryptError(Exception): + """ raise when unable to decrypt encrypted data + + """ + pass + + +class CryptoUnavailableError(Exception): + """ raise when Python Crypto module is not available + + """ + pass + + +def assert_crypto_availability(f): + """ Ensure Crypto module is available. """ + + @functools.wraps(f) + def wrapper(*args, **kwds): + if AES is None: + raise CryptoUnavailableError() + return f(*args, **kwds) + return wrapper + + +def generate_aes_key(token, secret): + """ Generates and returns a 256 bit AES key, based on sha256 hash. """ + return hashlib.sha256(token + secret).digest() + + +def compute_mac(token, serialized_data): + """ Computes and returns the base64 encoded MAC. """ + return hash_data(serialized_data + token) + + +def hash_data(data): + """ Return the base64 encoded SHA1 hash of the data. """ + return base64.b64encode(hashlib.sha1(data).digest()) + + +def sign_data(token, data): + """ MAC the data using SHA1. """ + mac_data = {} + mac_data['serialized_data'] = json.dumps(data) + mac = compute_mac(token, mac_data['serialized_data']) + mac_data['mac'] = mac + md = MAC_MARKER + base64.b64encode(json.dumps(mac_data)) + return md + + +def verify_signed_data(token, data): + """ Verify data integrity by ensuring MAC is valid. """ + if data.startswith(MAC_MARKER): + try: + data = data[len(MAC_MARKER):] + mac_data = json.loads(base64.b64decode(data)) + mac = compute_mac(token, mac_data['serialized_data']) + if mac != mac_data['mac']: + raise InvalidMacError('invalid MAC; expect=%s, actual=%s' % + (mac_data['mac'], mac)) + return json.loads(mac_data['serialized_data']) + except: + raise InvalidMacError('invalid MAC; data appeared to be corrupted') + else: + # doesn't appear to be MACed data + return data + + +@assert_crypto_availability +def encrypt_data(token, secret, data): + """ Encryptes the data with the given secret key. """ + iv = os.urandom(16) + aes_key = generate_aes_key(token, secret) + cipher = AES.new(aes_key, AES.MODE_CFB, iv) + data = json.dumps(data) + encoded_data = base64.b64encode(iv + cipher.encrypt(data)) + encoded_data = ENCRYPT_MARKER + encoded_data + return encoded_data + + +@assert_crypto_availability +def decrypt_data(token, secret, data): + """ Decrypt the data with the given secret key. """ + if data.startswith(ENCRYPT_MARKER): + try: + # encrypted data + encoded_data = data[len(ENCRYPT_MARKER):] + aes_key = generate_aes_key(token, secret) + decoded_data = base64.b64decode(encoded_data) + iv = decoded_data[:16] + encrypted_data = decoded_data[16:] + cipher = AES.new(aes_key, AES.MODE_CFB, iv) + decrypted_data = cipher.decrypt(encrypted_data) + return json.loads(decrypted_data) + except: + raise DecryptError('data appeared to be corrupted') + else: + # doesn't appear to be encrypted data + return data From 4123837cd619030fa8510838243d623478a85d7f Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Mon, 21 Jan 2013 17:15:28 +0100 Subject: [PATCH 013/120] Fix thinko in self.middleware.cert_file_missing The python function string.find() returns -1 on a miss, which is also evaluated as True. Therefore use the "X in Y" approach instead. Also added a rather trivial test to test for this code bug. In order to make the code easier to test, I've changed the parameters to operate on the command output, not the exception object and updated all callers. Change-Id: If0b4fed6fe676cad50512267c1b601a3a8a631e5 --- auth_token.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/auth_token.py b/auth_token.py index 7d332205..df2076fc 100644 --- a/auth_token.py +++ b/auth_token.py @@ -793,9 +793,8 @@ class AuthProtocol(object): 'Marking token %s as unauthorized in memcache', token) self._cache_store(token, 'invalid') - def cert_file_missing(self, called_proc_err, file_name): - return (called_proc_err.output.find(file_name) - and not os.path.exists(file_name)) + def cert_file_missing(self, proc_output, file_name): + return (file_name in proc_output and not os.path.exists(file_name)) def verify_uuid_token(self, user_token, retry=True): """Authenticate user token with keystone. @@ -867,10 +866,11 @@ class AuthProtocol(object): output = cms.cms_verify(data, self.signing_cert_file_name, self.ca_file_name) except cms.subprocess.CalledProcessError as err: - if self.cert_file_missing(err, self.signing_cert_file_name): + if self.cert_file_missing(err.output, + self.signing_cert_file_name): self.fetch_signing_cert() continue - if self.cert_file_missing(err, self.ca_file_name): + if self.cert_file_missing(err.output, self.ca_file_name): self.fetch_ca_cert() continue raise err From 08dec7dbe381a57437f239373d06e351e30a8407 Mon Sep 17 00:00:00 2001 From: Michael J Fork Date: Mon, 4 Feb 2013 15:42:58 +0000 Subject: [PATCH 014/120] Mark password config options with secret Config object supports masking values when writing out if the secret flag is set on the option definition. This change flags all python-keystoneclient options containing a password. Change-Id: I1ffc9727ab66c7acc6d4cf8e5870bb32f5e68b67 --- auth_token.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index df2076fc..06e26389 100644 --- a/auth_token.py +++ b/auth_token.py @@ -161,9 +161,9 @@ opts = [ cfg.StrOpt('auth_protocol', default='https'), cfg.StrOpt('auth_uri', default=None), cfg.BoolOpt('delay_auth_decision', default=False), - cfg.StrOpt('admin_token'), + cfg.StrOpt('admin_token', secret=True), cfg.StrOpt('admin_user'), - cfg.StrOpt('admin_password'), + cfg.StrOpt('admin_password', secret=True), cfg.StrOpt('admin_tenant_name', default='admin'), cfg.StrOpt('cache', default=None), # env key for the swift cache cfg.StrOpt('certfile'), @@ -173,7 +173,7 @@ opts = [ cfg.ListOpt('memcache_servers'), cfg.IntOpt('token_cache_time', default=300), cfg.StrOpt('memcache_security_strategy', default=None), - cfg.StrOpt('memcache_secret_key', default=None), + cfg.StrOpt('memcache_secret_key', default=None, secret=True), ] CONF.register_opts(opts, group='keystone_authtoken') From 8003e7bfa9fdd82c96607fb23761079aca346363 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 14 Feb 2013 14:43:54 +0100 Subject: [PATCH 015/120] Allow configure auth_token http connect timeout. - Fixes bug 939613. Change-Id: Ic8cfc36e02212eeb987e509893369c0a47d9209a --- auth_token.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 06e26389..893738b9 100644 --- a/auth_token.py +++ b/auth_token.py @@ -161,6 +161,7 @@ opts = [ cfg.StrOpt('auth_protocol', default='https'), cfg.StrOpt('auth_uri', default=None), cfg.BoolOpt('delay_auth_decision', default=False), + cfg.BoolOpt('http_connect_timeout', default=None), cfg.StrOpt('admin_token', secret=True), cfg.StrOpt('admin_user'), cfg.StrOpt('admin_password', secret=True), @@ -287,6 +288,9 @@ class AuthProtocol(object): self._token_revocation_list_fetched_time = None cache_timeout = datetime.timedelta(seconds=0) self.token_revocation_list_cache_timeout = cache_timeout + http_connect_timeout_cfg = self._conf_get('http_connect_timeout') + self.http_connect_timeout = (http_connect_timeout_cfg and + int(http_connect_timeout_cfg)) def _assert_valid_memcache_protection_config(self): if self._memcache_security_strategy: @@ -439,12 +443,14 @@ class AuthProtocol(object): def _get_http_connection(self): if self.auth_protocol == 'http': - return self.http_client_class(self.auth_host, self.auth_port) + return self.http_client_class(self.auth_host, self.auth_port, + timeout=self.http_connect_timeout) else: return self.http_client_class(self.auth_host, self.auth_port, self.key_file, - self.cert_file) + self.cert_file, + timeout=self.http_connect_timeout) def _http_request(self, method, path): """HTTP request helper used to make unspecified content type requests. From eb0f676686989cf66c3115b6ed0794a075fbfa21 Mon Sep 17 00:00:00 2001 From: Alan Pevec Date: Sun, 17 Feb 2013 00:05:40 +0100 Subject: [PATCH 016/120] Use oslo-config-2013.1b3 The cfg API is now available via the oslo-config library, so switch to it and remove the copied-and-pasted version. Removes the load of copied-and-pasted cfg from each application in authtoken middleware. Depends on oslo-config deployments: Nova I4815aeb8a9341a31a250e920157f15ee15cfc5bc Glance I4815aeb8a9341a31a250e920157f15ee15cfc5bc Quantum I4815aeb8a9341a31a250e920157f15ee15cfc5bc Cinder I4815aeb8a9341a31a250e920157f15ee15cfc5bc Add the 2013.1b3 tarball to tools/pip-requires - this will be changed to 'oslo-config>=2013.1' when oslo-config is published to pypi. This will happen in time for grizzly final. Change-Id: I18c450174277c8e2d15ed93879da6cd92074c27a --- auth_token.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/auth_token.py b/auth_token.py index 893738b9..5d1d33d2 100644 --- a/auth_token.py +++ b/auth_token.py @@ -119,23 +119,20 @@ from keystoneclient.middleware import memcache_crypt from keystoneclient.openstack.common import timeutils CONF = None -try: - from openstack.common import cfg - CONF = cfg.CONF -except ImportError: - # cfg is not a library yet, try application copies - for app in 'nova', 'glance', 'quantum', 'cinder': - try: - cfg = __import__('%s.openstack.common.cfg' % app, - fromlist=['%s.openstack.common' % app]) - # test which application middleware is running in - if hasattr(cfg, 'CONF') and 'config_file' in cfg.CONF: - CONF = cfg.CONF - break - except ImportError: - pass +# to pass gate before oslo-config is deployed everywhere, +# try application copies first +for app in 'nova', 'glance', 'quantum', 'cinder': + try: + cfg = __import__('%s.openstack.common.cfg' % app, + fromlist=['%s.openstack.common' % app]) + # test which application middleware is running in + if hasattr(cfg, 'CONF') and 'config_file' in cfg.CONF: + CONF = cfg.CONF + break + except ImportError: + pass if not CONF: - from keystoneclient.openstack.common import cfg + from oslo.config import cfg CONF = cfg.CONF # alternative middleware configuration in the main application's From c32de9b9dc800148f3c448c3a6e789065ffae05d Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Wed, 13 Feb 2013 10:05:49 -0600 Subject: [PATCH 017/120] Remove test dep on name of dir (bug 1124283) The name of the project's directory was previously hardcoded into the tests, so tests would fail if executed from another directory (such as "python-keystoneclient-master", as checked-out by keystone for integration testing). Also, the tests should now be executable on Windows. Change-Id: I0a1e052054e509b0f795fd13f95a804e0c255907 --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index e5c11712..77511412 100644 --- a/test.py +++ b/test.py @@ -22,7 +22,7 @@ import os import sys -ROOTDIR = os.path.dirname(os.path.abspath(os.curdir)) +ROOTDIR = os.path.abspath(os.curdir) def rootdir(*p): From 4b63579b4d38093901612c91d9f0a2ecc38d7fa6 Mon Sep 17 00:00:00 2001 From: Henry Nash Date: Mon, 4 Mar 2013 05:05:15 +0000 Subject: [PATCH 018/120] Fix auth-token middleware to understand v3 tokens Now that the Identity server supports v3 tokens, the auth_token middleware should permit the in-line validation of such a token. This essentially means just setting any new environment items that correspond to the new attributes that may be in a v3 token (such as domains), as well as allowing for the slight format differences. Most of the work in this change is actually in the unit tests, where it was important to try and enable the existing tests to be run against an auth_token middleware configured for both v2 and v3. This meant restructing the test class so that the token format is separated from the individual tests and is initialized by the class Setup(). Since there are some new signed token formats included in this testing, a new set of the signed tokens was generated. Fixes Bug #1132390 Change-Id: I78b232d30f5310c39089fbbc8e56c23df291f89f --- auth_token.py | 255 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 217 insertions(+), 38 deletions(-) diff --git a/auth_token.py b/auth_token.py index 5d1d33d2..084d6f2c 100644 --- a/auth_token.py +++ b/auth_token.py @@ -61,35 +61,69 @@ HTTP_X_IDENTITY_STATUS The underlying service will only see a value of 'Invalid' if the Middleware is configured to run in 'delay_auth_decision' mode -HTTP_X_TENANT_ID - Identity service managed unique identifier, string +HTTP_X_DOMAIN_ID + Identity service managed unique identifier, string. Only present if + this is a domain-scoped token. -HTTP_X_TENANT_NAME - Unique tenant identifier, string +HTTP_X_DOMAIN_NAME + Unique domain name, string. Only present if this is a domain-scoped token. + +HTTP_X_PROJECT_ID + Identity service managed unique identifier, string. Only present if + this is a project-scoped token. + +HTTP_X_PROJECT_NAME + Project name, unique within owning domain, string. Only present if + this is a project-scoped token. + +HTTP_X_PROJECT_DOMAIN_ID + Identity service managed unique identifier of owning domain of + project, string. Only present if this is a project-scoped token. + +HTTP_X_PROJECT_DOMAIN_NAME + Name of owning domain of project, string. Only present if this is a + project-scoped token. HTTP_X_USER_ID Identity-service managed unique identifier, string HTTP_X_USER_NAME - Unique user identifier, string + User identifier, unique within owning domain, string + +HTTP_X_USER_DOMAIN_ID + Identity service managed unique identifier of owning domain of user, string + +HTTP_X_USER_DOMAIN_NAME + Name of owning domain of user, string HTTP_X_ROLES - Comma delimited list of case-sensitive Roles + Comma delimited list of case-sensitive role names HTTP_X_SERVICE_CATALOG json encoded keystone service catalog (optional). +HTTP_X_TENANT_ID + *Deprecated* in favor of HTTP_X_PROJECT_ID + Identity service managed unique identifier, string. For v3 tokens, this + will be set to the same value as HTTP_X_PROJECT_ID + +HTTP_X_TENANT_NAME + *Deprecated* in favor of HTTP_X_PROJECT_NAME + Project identifier, unique within owning domain, string. For v3 tokens, + this will be set to the same value as HTTP_X_PROJECT_NAME + HTTP_X_TENANT *Deprecated* in favor of HTTP_X_TENANT_ID and HTTP_X_TENANT_NAME - Keystone-assigned unique identifier, deprecated + Keystone-assigned unique identifier, string. For v3 tokens, this + will be set to the same value as HTTP_X_PROJECT_ID HTTP_X_USER *Deprecated* in favor of HTTP_X_USER_ID and HTTP_X_USER_NAME - Unique user name, string + User name, unique within owning domain, string HTTP_X_ROLE *Deprecated* in favor of HTTP_X_ROLES - This is being renamed, and the new header contains the same data. + Will contain the same values as HTTP_X_ROLES. OTHER ENVIRONMENT VARIABLES --------------------------- @@ -157,8 +191,10 @@ opts = [ cfg.IntOpt('auth_port', default=35357), cfg.StrOpt('auth_protocol', default='https'), cfg.StrOpt('auth_uri', default=None), + cfg.StrOpt('auth_version', default=None), cfg.BoolOpt('delay_auth_decision', default=False), cfg.BoolOpt('http_connect_timeout', default=None), + cfg.StrOpt('http_handler', default=None), cfg.StrOpt('admin_token', secret=True), cfg.StrOpt('admin_user'), cfg.StrOpt('admin_password', secret=True), @@ -171,10 +207,12 @@ opts = [ cfg.ListOpt('memcache_servers'), cfg.IntOpt('token_cache_time', default=300), cfg.StrOpt('memcache_security_strategy', default=None), - cfg.StrOpt('memcache_secret_key', default=None, secret=True), + cfg.StrOpt('memcache_secret_key', default=None, secret=True) ] CONF.register_opts(opts, group='keystone_authtoken') +LIST_OF_VERSIONS_TO_ATTEMPT = ['v3.0', 'v2.0'] + def will_expire_soon(expiry): """ Determines if expiration is about to occur. @@ -221,10 +259,17 @@ class AuthProtocol(object): self.auth_host = self._conf_get('auth_host') self.auth_port = int(self._conf_get('auth_port')) self.auth_protocol = self._conf_get('auth_protocol') - if self.auth_protocol == 'http': - self.http_client_class = httplib.HTTPConnection + if not self._conf_get('http_handler'): + if self.auth_protocol == 'http': + self.http_client_class = httplib.HTTPConnection + else: + self.http_client_class = httplib.HTTPSConnection else: - self.http_client_class = httplib.HTTPSConnection + # Really only used for unit testing, since we need to + # have a fake handler set up before we issue an http + # request to get the list of versions supported by the + # server at the end of this initialization + self.http_client_class = self._conf_get('http_handler') self.auth_admin_prefix = self._conf_get('auth_admin_prefix') self.auth_uri = self._conf_get('auth_uri') @@ -289,6 +334,9 @@ class AuthProtocol(object): self.http_connect_timeout = (http_connect_timeout_cfg and int(http_connect_timeout_cfg)) + # Determine the highest api version we can use. + self.auth_version = self._choose_api_version() + def _assert_valid_memcache_protection_config(self): if self._memcache_security_strategy: if self._memcache_security_strategy not in ('MAC', 'ENCRYPT'): @@ -326,6 +374,60 @@ class AuthProtocol(object): else: return CONF.keystone_authtoken[name] + def _choose_api_version(self): + """ Determine the api version that we should use.""" + + # If the configuration specifies an auth_version we will just + # assume that is correct and use it. We could, of course, check + # that this version is supported by the server, but in case + # there are some problems in the field, we want as little code + # as possible in the way of letting auth_token talk to the + # server. + if self._conf_get('auth_version'): + version_to_use = self._conf_get('auth_version') + self.LOG.info('Auth Token proceeding with requested %s apis', + version_to_use) + else: + version_to_use = None + versions_supported_by_server = self._get_supported_versions() + if versions_supported_by_server: + for version in LIST_OF_VERSIONS_TO_ATTEMPT: + if version in versions_supported_by_server: + version_to_use = version + break + if version_to_use: + self.LOG.info('Auth Token confirmed use of %s apis', + version_to_use) + else: + self.LOG.error( + 'Attempted versions [%s] not in list supported by ' + 'server [%s]', + ', '.join(LIST_OF_VERSIONS_TO_ATTEMPT), + ', '.join(versions_supported_by_server)) + raise ServiceError('No compatible apis supported by server') + return version_to_use + + def _get_supported_versions(self): + versions = [] + response, data = self._json_request('GET', '/') + if response.status != 300: + self.LOG.error('Unable to get version info from keystone: %s' % + response.status) + raise ServiceError('Unable to get version info from keystone') + else: + try: + for version in data['versions']['values']: + versions.append(version['id']) + except KeyError: + self.LOG.error( + 'Invalid version response format from server', data) + raise ServiceError('Unable to parse version response ' + 'from keystone') + + self.LOG.debug('Server reports support for api versions: %s', + ', '.join(versions)) + return versions + def __call__(self, env, start_response): """Handle incoming request. @@ -371,14 +473,22 @@ class AuthProtocol(object): """ auth_headers = ( 'X-Identity-Status', - 'X-Tenant-Id', - 'X-Tenant-Name', + 'X-Domain-Id', + 'X-Domain-Name', + 'X-Project-Id', + 'X-Project-Name', + 'X-Project-Domain-Id', + 'X-Project-Domain-Name', 'X-User-Id', 'X-User-Name', + 'X-User-Domain-Id', + 'X-User-Domain-Name', 'X-Roles', 'X-Service-Catalog', # Deprecated 'X-User', + 'X-Tenant-Id', + 'X-Tenant-Name', 'X-Tenant', 'X-Role', ) @@ -459,7 +569,6 @@ class AuthProtocol(object): """ conn = self._get_http_connection() - try: conn.request(method, path) response = conn.getresponse() @@ -509,7 +618,6 @@ class AuthProtocol(object): raise ServiceError('Unable to communicate with keystone') finally: conn.close() - try: data = jsonutils.loads(body) except ValueError: @@ -524,6 +632,10 @@ class AuthProtocol(object): :return token id upon success :raises ServerError when unable to communicate with keystone + Irrespective of the auth version we are going to use for the + user token, for simplicity we always use a v2 admin token to + validate the user token. + """ params = { 'auth': { @@ -588,26 +700,35 @@ class AuthProtocol(object): Build headers that represent authenticated user: * X_IDENTITY_STATUS: Confirmed or Invalid - * X_TENANT_ID: id of tenant if tenant is present - * X_TENANT_NAME: name of tenant if tenant is present + * X_DOMAIN_ID: id of domain, if token is scoped to a domain + * X_DOMAIN_NAME: name of domain, if token is scoped to a domain + * X_PROJECT_ID: id of project, if token is scoped to a project + * X_PROJECT_NAME: name of project, if token is scoped to a project + * X_PROJECT_DOMAIN_ID: id of owning domain of project, if + token is scoped to a project + * X_PROJECT_DOMAIN_NAME: name of owning domain of project, if + token is scoped to a project * X_USER_ID: id of user * X_USER_NAME: name of user + * X_USER_DOMAIN_ID: id of owning domain of user + * X_USER_DOMAIN_NAME: name of owning domain of user * X_ROLES: list of roles * X_SERVICE_CATALOG: service catalog - Additional (deprecated) headers include: + Additional (deprecated) headers: * X_USER: name of user - * X_TENANT: For legacy compatibility before we had ID and Name + * X_TENANT_ID: id of tenant (which is equivilent to project), + if token is scoped to a project + * X_TENANT_NAME: name of tenant (which is equivilent to project), + if token is scoped to a project + * X_TENANT: For legacy compatibility before we had ID and Name, this + is will be the same as X_TENANT_NAME * X_ROLE: list of roles :param token_info: token object returned by keystone on authentication :raise InvalidUserToken when unable to parse token object """ - user = token_info['access']['user'] - token = token_info['access']['token'] - roles = ','.join([role['name'] for role in user.get('roles', [])]) - def get_tenant_info(): """Returns a (tenant_id, tenant_name) tuple from context.""" def essex(): @@ -619,7 +740,7 @@ class AuthProtocol(object): return (token['tenantId'], token['tenantId']) def default_tenant(): - """Assume the user's default tenant.""" + """Pre-grizzly, assume the user's default tenant.""" return (user['tenantId'], user['tenantName']) for method in [essex, pre_diablo, default_tenant]: @@ -630,26 +751,72 @@ class AuthProtocol(object): raise InvalidUserToken('Unable to determine tenancy.') - tenant_id, tenant_name = get_tenant_info() + # For clarity. set all those attributes that are optional in + # either a v2 or v3 token to None first + domain_id = None + domain_name = None + project_id = None + project_name = None + user_domain_id = None + user_domain_name = None + project_domain_id = None + project_domain_name = None + + if 'access' in token_info: + #v2 token + user = token_info['access']['user'] + token = token_info['access']['token'] + roles = ','.join([role['name'] for role in user.get('roles', [])]) + catalog_root = token_info['access'] + catalog_key = 'serviceCatalog' + project_id, project_name = get_tenant_info() + else: + #v3 token + token = token_info['token'] + user = token['user'] + user_domain_id = user['domain']['id'] + user_domain_name = user['domain']['name'] + roles = (','.join([role['name'] + for role in token.get('roles', [])])) + catalog_root = token + catalog_key = 'catalog' + # For v3, the server will put in the default project if there is + # one, so no need for us to add it here (like we do for a v2 token) + if 'domain' in token: + domain_id = token['domain']['id'] + domain_name = token['domain']['name'] + elif 'project' in token: + project_id = token['project']['id'] + project_name = token['project']['name'] + project_domain_id = token['project']['domain']['id'] + project_domain_name = token['project']['domain']['name'] user_id = user['id'] user_name = user['name'] rval = { 'X-Identity-Status': 'Confirmed', - 'X-Tenant-Id': tenant_id, - 'X-Tenant-Name': tenant_name, + 'X-Domain-Id': domain_id, + 'X-Domain-Name': domain_name, + 'X-Project-Id': project_id, + 'X-Project-Name': project_name, + 'X-Project-Domain-Id': project_domain_id, + 'X-Project-Domain-Name': project_domain_name, 'X-User-Id': user_id, 'X-User-Name': user_name, + 'X-User-Domain-Id': user_domain_id, + 'X-User-Domain-Name': user_domain_name, 'X-Roles': roles, # Deprecated 'X-User': user_name, - 'X-Tenant': tenant_name, + 'X-Tenant-Id': project_id, + 'X-Tenant-Name': project_name, + 'X-Tenant': project_name, 'X-Role': roles, } try: - catalog = token_info['access']['serviceCatalog'] + catalog = catalog_root[catalog_key] rval['X-Service-Catalog'] = jsonutils.dumps(catalog) except KeyError: pass @@ -781,11 +948,15 @@ class AuthProtocol(object): """ if self._cache and data: if 'token' in data.get('access', {}): + # It's a v2 token timestamp = data['access']['token']['expires'] - expires = timeutils.parse_isotime(timestamp).strftime('%s') + elif 'token' in data: + # It's a v3 token + timestamp = data['token']['expires'] else: self.LOG.error('invalid token format') return + expires = timeutils.parse_isotime(timestamp).strftime('%s') self.LOG.debug('Storing %s token in memcache', token) self._cache_store(token, data, expires) @@ -811,12 +982,19 @@ class AuthProtocol(object): :raise ServiceError if unable to authenticate token """ - - headers = {'X-Auth-Token': self.get_admin_token()} - response, data = self._json_request( - 'GET', - '/v2.0/tokens/%s' % safe_quote(user_token), - additional_headers=headers) + if self.auth_version == 'v3.0': + headers = {'X-Auth-Token': self.get_admin_token(), + 'X-Subject-Token': safe_quote(user_token)} + response, data = self._json_request( + 'GET', + '/v3/auth/tokens', + additional_headers=headers) + else: + headers = {'X-Auth-Token': self.get_admin_token()} + response, data = self._json_request( + 'GET', + '/v2.0/tokens/%s' % safe_quote(user_token), + additional_headers=headers) if response.status == 200: self._cache_put(user_token, data) @@ -910,6 +1088,7 @@ class AuthProtocol(object): timeout = (self.token_revocation_list_fetched_time + self.token_revocation_list_cache_timeout) list_is_current = timeutils.utcnow() < timeout + if list_is_current: # Load the list from disk if required if not self._token_revocation_list: From cc2e01bfb3cd81d2297dc31028bdc2e4a9c881f4 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Tue, 12 Mar 2013 11:24:35 -0400 Subject: [PATCH 019/120] Use v2.0 api by default in auth_token middleware Fixes an issue that crept in with d782a99 where auth_token started defaulting to the v3.0 API by default when no version was specified. Given that bin/keystone still defaults to using the v2.0 API it seems like auth_token.py should too. Fixes LP Bug #1154144 Change-Id: Ia5620bccc182bbc73cb60dcccb1f701304450e5a --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 084d6f2c..0a28194b 100644 --- a/auth_token.py +++ b/auth_token.py @@ -211,7 +211,7 @@ opts = [ ] CONF.register_opts(opts, group='keystone_authtoken') -LIST_OF_VERSIONS_TO_ATTEMPT = ['v3.0', 'v2.0'] +LIST_OF_VERSIONS_TO_ATTEMPT = ['v2.0', 'v3.0'] def will_expire_soon(expiry): From 13adb35e717e10f9b0e392d9234bffc415f4cbf7 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 6 Mar 2013 12:10:25 -0800 Subject: [PATCH 020/120] Retry http_request and json_request failure. Temporary network outages or keystone server restarts can lead to a connection refused in auth_token middleware. Rather than failing immediately, this patch attempts to retry a few times. Fixes bug 1150299 Change-Id: I2ecf0d7745290976efcb3e3cd6511817a53d3e0a --- auth_token.py | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/auth_token.py b/auth_token.py index 084d6f2c..df656897 100644 --- a/auth_token.py +++ b/auth_token.py @@ -559,25 +559,36 @@ class AuthProtocol(object): self.cert_file, timeout=self.http_connect_timeout) - def _http_request(self, method, path): + def _http_request(self, method, path, **kwargs): """HTTP request helper used to make unspecified content type requests. :param method: http method :param path: relative request url - :return (http response object) + :return (http response object, response body) :raise ServerError when unable to communicate with keystone """ conn = self._get_http_connection() - try: - conn.request(method, path) - response = conn.getresponse() - body = response.read() - except Exception as e: - self.LOG.error('HTTP connection exception: %s' % e) - raise ServiceError('Unable to communicate with keystone') - finally: - conn.close() + + RETRIES = 3 + retry = 0 + + while True: + try: + conn.request(method, path, **kwargs) + response = conn.getresponse() + body = response.read() + break + except Exception as e: + if retry == RETRIES: + self.LOG.error('HTTP connection exception: %s' % e) + raise ServiceError('Unable to communicate with keystone') + # NOTE(vish): sleep 0.5, 1, 2 + self.LOG.warn('Retrying on HTTP connection exception: %s' % e) + time.sleep(2.0 ** retry / 2) + retry += 1 + finally: + conn.close() return response, body @@ -593,8 +604,6 @@ class AuthProtocol(object): :raise ServerError when unable to communicate with keystone """ - conn = self._get_http_connection() - kwargs = { 'headers': { 'Content-type': 'application/json', @@ -608,16 +617,10 @@ class AuthProtocol(object): if body: kwargs['body'] = jsonutils.dumps(body) - full_path = self.auth_admin_prefix + path - try: - conn.request(method, full_path, **kwargs) - response = conn.getresponse() - body = response.read() - except Exception as e: - self.LOG.error('HTTP connection exception: %s' % e) - raise ServiceError('Unable to communicate with keystone') - finally: - conn.close() + path = self.auth_admin_prefix + path + + response, body = self._http_request(method, path, **kwargs) + try: data = jsonutils.loads(body) except ValueError: From 3b2d1af4c19078502bfe07ca50c2a552c9347d41 Mon Sep 17 00:00:00 2001 From: Henry Nash Date: Wed, 13 Mar 2013 21:58:14 +0000 Subject: [PATCH 021/120] Doc info and other readability improvements A few good suggestions were made on the final round of reviews on the previous patch to add v3 token support, but were not implemented. This patch applies these, which should not create any functional change. Fixes Bug #1154768 Change-Id: Ie5408a5477d176bd28b2c385e49cd29b39c0de39 --- auth_token.py | 67 +++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/auth_token.py b/auth_token.py index 084d6f2c..c21cb260 100644 --- a/auth_token.py +++ b/auth_token.py @@ -63,26 +63,30 @@ HTTP_X_IDENTITY_STATUS HTTP_X_DOMAIN_ID Identity service managed unique identifier, string. Only present if - this is a domain-scoped token. + this is a domain-scoped v3 token. HTTP_X_DOMAIN_NAME - Unique domain name, string. Only present if this is a domain-scoped token. + Unique domain name, string. Only present if this is a domain-scoped + v3 token. HTTP_X_PROJECT_ID Identity service managed unique identifier, string. Only present if - this is a project-scoped token. + this is a project-scoped v3 token, or a tenant-scoped v2 token. HTTP_X_PROJECT_NAME Project name, unique within owning domain, string. Only present if - this is a project-scoped token. + this is a project-scoped v3 token, or a tenant-scoped v2 token. HTTP_X_PROJECT_DOMAIN_ID Identity service managed unique identifier of owning domain of - project, string. Only present if this is a project-scoped token. + project, string. Only present if this is a project-scoped v3 token. If + this variable is set, this indicates that the PROJECT_NAME can only + be assumed to be unique within this domain. HTTP_X_PROJECT_DOMAIN_NAME Name of owning domain of project, string. Only present if this is a - project-scoped token. + project-scoped v3 token. If this variable is set, this indicates that + the PROJECT_NAME can only be assumed to be unique within this domain. HTTP_X_USER_ID Identity-service managed unique identifier, string @@ -91,10 +95,14 @@ HTTP_X_USER_NAME User identifier, unique within owning domain, string HTTP_X_USER_DOMAIN_ID - Identity service managed unique identifier of owning domain of user, string + Identity service managed unique identifier of owning domain of + user, string. If this variable is set, this indicates that the USER_NAME + can only be assumed to be unique within this domain. HTTP_X_USER_DOMAIN_NAME - Name of owning domain of user, string + Name of owning domain of user, string. If this variable is set, this + indicates that the USER_NAME can only be assumed to be unique within + this domain. HTTP_X_ROLES Comma delimited list of case-sensitive role names @@ -695,35 +703,17 @@ class AuthProtocol(object): self.LOG.warn("Authorization failed for token %s", user_token) raise InvalidUserToken('Token authorization failed') + def _token_is_v2(self, token_info): + return ('access' in token_info) + + def _token_is_v3(self, token_info): + return ('token' in token_info) + def _build_user_headers(self, token_info): """Convert token object into headers. - Build headers that represent authenticated user: - * X_IDENTITY_STATUS: Confirmed or Invalid - * X_DOMAIN_ID: id of domain, if token is scoped to a domain - * X_DOMAIN_NAME: name of domain, if token is scoped to a domain - * X_PROJECT_ID: id of project, if token is scoped to a project - * X_PROJECT_NAME: name of project, if token is scoped to a project - * X_PROJECT_DOMAIN_ID: id of owning domain of project, if - token is scoped to a project - * X_PROJECT_DOMAIN_NAME: name of owning domain of project, if - token is scoped to a project - * X_USER_ID: id of user - * X_USER_NAME: name of user - * X_USER_DOMAIN_ID: id of owning domain of user - * X_USER_DOMAIN_NAME: name of owning domain of user - * X_ROLES: list of roles - * X_SERVICE_CATALOG: service catalog - - Additional (deprecated) headers: - * X_USER: name of user - * X_TENANT_ID: id of tenant (which is equivilent to project), - if token is scoped to a project - * X_TENANT_NAME: name of tenant (which is equivilent to project), - if token is scoped to a project - * X_TENANT: For legacy compatibility before we had ID and Name, this - is will be the same as X_TENANT_NAME - * X_ROLE: list of roles + Build headers that represent authenticated user - see main + doc info at start of file for details of headers to be defined. :param token_info: token object returned by keystone on authentication :raise InvalidUserToken when unable to parse token object @@ -762,8 +752,7 @@ class AuthProtocol(object): project_domain_id = None project_domain_name = None - if 'access' in token_info: - #v2 token + if self._token_is_v2(token_info): user = token_info['access']['user'] token = token_info['access']['token'] roles = ','.join([role['name'] for role in user.get('roles', [])]) @@ -947,11 +936,9 @@ class AuthProtocol(object): quick check of token freshness on retrieval. """ if self._cache and data: - if 'token' in data.get('access', {}): - # It's a v2 token + if self._token_is_v2(data): timestamp = data['access']['token']['expires'] - elif 'token' in data: - # It's a v3 token + elif self._token_is_v3(data): timestamp = data['token']['expires'] else: self.LOG.error('invalid token format') From eabcd39b8dfec781ae754c3119a722dfa0b06c33 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 13 Mar 2013 17:33:22 -0400 Subject: [PATCH 022/120] Make auth_token lazy load the auth_version. Updates recent changes to the auth_token middleware (this is a regression in d782a99) so that self.auth_version is lazy loaded. This fixes issues where other openstack services would fail to startup correctly if Keystone is not running. The issue was auth_token was trying to make a request to '/' to get versions information on startup to "autodetect" the correct version to use. This patch fixes startup issues by moving the version detection so that it is lazy loaded right before it is actually used. This issue should fix SmokeStack :) Fixes LP Bug #1154806. Change-Id: Ib24f5386fa1ffe0e0365548840f0cfeaae36f548 --- auth_token.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index 084d6f2c..f5181d3f 100644 --- a/auth_token.py +++ b/auth_token.py @@ -333,9 +333,7 @@ class AuthProtocol(object): http_connect_timeout_cfg = self._conf_get('http_connect_timeout') self.http_connect_timeout = (http_connect_timeout_cfg and int(http_connect_timeout_cfg)) - - # Determine the highest api version we can use. - self.auth_version = self._choose_api_version() + self.auth_version = None def _assert_valid_memcache_protection_config(self): if self._memcache_security_strategy: @@ -982,6 +980,10 @@ class AuthProtocol(object): :raise ServiceError if unable to authenticate token """ + # Determine the highest api version we can use. + if not self.auth_version: + self.auth_version = self._choose_api_version() + if self.auth_version == 'v3.0': headers = {'X-Auth-Token': self.get_admin_token(), 'X-Subject-Token': safe_quote(user_token)} From 9a1db28c3854be3627ab4f8ea9eb101891661dcf Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 1 Mar 2013 10:15:26 -0800 Subject: [PATCH 023/120] Cache tokens using memorycache from oslo. Verifying tokens accounts for a significant portion of the time taken by the various api servers. This uses the simple memory cache from oslo so that there is a simple default cache even if memcached is not deployed. Also cleans up the tests and removes unnecessary fakes. DocImpact Change-Id: I501c14f2f51da058cb574c32d49dd769e6f6ad86 --- auth_token.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/auth_token.py b/auth_token.py index 399b116a..7dfca551 100644 --- a/auth_token.py +++ b/auth_token.py @@ -150,6 +150,7 @@ from keystoneclient.openstack.common import jsonutils from keystoneclient.common import cms from keystoneclient import utils from keystoneclient.middleware import memcache_crypt +from keystoneclient.openstack.common import memorycache from keystoneclient.openstack.common import timeutils CONF = None @@ -353,16 +354,8 @@ class AuthProtocol(object): self._cache = env.get(cache) else: # use Keystone memcache - memcache_servers = self._conf_get('memcache_servers') - if memcache_servers: - try: - import memcache - self.LOG.info('Using Keystone memcache for caching token') - self._cache = memcache.Client(memcache_servers) - self._use_keystone_cache = True - except ImportError as e: - msg = 'disabled caching due to missing libraries %s' % (e) - self.LOG.warn(msg) + self._cache = memorycache.get_client(memcache_servers) + self._use_keystone_cache = True self._cache_initialized = True def _conf_get(self, name): @@ -999,12 +992,8 @@ class AuthProtocol(object): additional_headers=headers) if response.status == 200: - self._cache_put(user_token, data) return data if response.status == 404: - # FIXME(ja): I'm assuming the 404 status means that user_token is - # invalid - not that the admin_token is invalid - self._cache_store_invalid(user_token) self.LOG.warn("Authorization failed for token %s", user_token) raise InvalidUserToken('Token authorization failed') if response.status == 401: From f21cd7ceca22219a36a4622cecdeec87a7179650 Mon Sep 17 00:00:00 2001 From: Adam Young Date: Tue, 19 Mar 2013 13:20:17 -0400 Subject: [PATCH 024/120] Config value for revocation list timeout Adds the config option 'revocation_cache_time' default of 300 seconds, same as token timeout Bug 1076083 DocImpact Change-Id: Ifd41c816dd5431f140461d6a1588364d7ecf9a62 --- auth_token.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 98a427bc..694089c9 100644 --- a/auth_token.py +++ b/auth_token.py @@ -215,6 +215,7 @@ opts = [ default=os.path.expanduser('~/keystone-signing')), cfg.ListOpt('memcache_servers'), cfg.IntOpt('token_cache_time', default=300), + cfg.IntOpt('revocation_cache_time', default=1), cfg.StrOpt('memcache_security_strategy', default=None), cfg.StrOpt('memcache_secret_key', default=None, secret=True) ] @@ -337,8 +338,8 @@ class AuthProtocol(object): self.token_cache_time = int(self._conf_get('token_cache_time')) self._token_revocation_list = None self._token_revocation_list_fetched_time = None - cache_timeout = datetime.timedelta(seconds=0) - self.token_revocation_list_cache_timeout = cache_timeout + self.token_revocation_list_cache_timeout = datetime.timedelta( + seconds=self._conf_get('revocation_cache_time')) http_connect_timeout_cfg = self._conf_get('http_connect_timeout') self.http_connect_timeout = (http_connect_timeout_cfg and int(http_connect_timeout_cfg)) From 2fb63d56e8f3b3875c68c53b82662a44f72678f5 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 25 Mar 2013 13:19:26 -0400 Subject: [PATCH 025/120] Allow keystoneclient to work with older keystone installs Older keystone installs may return 501 for GET / instead of 300 like today. We should be able to assume that if 501 is returned the server will support v2.0 only. Fixes bug 1159911 Change-Id: If264840d8678a490264f1bdb62f1b51c362619e1 --- auth_token.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 694089c9..f86359fd 100644 --- a/auth_token.py +++ b/auth_token.py @@ -410,7 +410,10 @@ class AuthProtocol(object): def _get_supported_versions(self): versions = [] response, data = self._json_request('GET', '/') - if response.status != 300: + if response.status == 501: + self.LOG.warning("Old keystone installation found...assuming v2.0") + versions.append("v2.0") + elif response.status != 300: self.LOG.error('Unable to get version info from keystone: %s' % response.status) raise ServiceError('Unable to get version info from keystone') From c07213637c7225115f77695f2a2ffaab167cdfab Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Tue, 12 Mar 2013 15:54:51 +0000 Subject: [PATCH 026/120] Fix v3 with UUID and memcache expiring. - Regenerate tokens to change expires in expires_at. Change-Id: Iaa62dca50d34a228e4850b59d263b807c5ee3549 --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 694089c9..0d0e1244 100644 --- a/auth_token.py +++ b/auth_token.py @@ -934,7 +934,7 @@ class AuthProtocol(object): if self._token_is_v2(data): timestamp = data['access']['token']['expires'] elif self._token_is_v3(data): - timestamp = data['token']['expires'] + timestamp = data['token']['expires_at'] else: self.LOG.error('invalid token format') return From c9bd6d08684c69db2a8d20b354b9ae060a72b9c6 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Wed, 8 May 2013 10:49:20 -0500 Subject: [PATCH 027/120] Securely create signing_dir (bug 1174608) Also verifies the security of an existing signing_dir. Change-Id: I0685b4274a94ad3974a2b2a7ab3f45830d3934bb --- auth_token.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/auth_token.py b/auth_token.py index 0d0e1244..e6cf99f4 100644 --- a/auth_token.py +++ b/auth_token.py @@ -296,15 +296,20 @@ class AuthProtocol(object): self.signing_dirname = self._conf_get('signing_dir') self.LOG.info('Using %s as cache directory for signing certificate' % self.signing_dirname) - if (os.path.exists(self.signing_dirname) and - not os.access(self.signing_dirname, os.W_OK)): - raise ConfigurationError("unable to access signing dir %s" % - self.signing_dirname) - - if not os.path.exists(self.signing_dirname): - os.makedirs(self.signing_dirname) - #will throw IOError if it cannot change permissions - os.chmod(self.signing_dirname, stat.S_IRWXU) + if os.path.exists(self.signing_dirname): + if not os.access(self.signing_dirname, os.W_OK): + raise ConfigurationError( + 'unable to access signing_dir %s' % self.signing_dirname) + if os.stat(self.signing_dirname).st_uid != os.getuid(): + self.LOG.warning( + 'signing_dir is not owned by %s' % os.getlogin()) + current_mode = stat.S_IMODE(os.stat(self.signing_dirname).st_mode) + if current_mode != stat.S_IRWXU: + self.LOG.warning( + 'signing_dir mode is %s instead of %s' % + (oct(current_mode), oct(stat.S_IRWXU))) + else: + os.makedirs(self.signing_dirname, stat.S_IRWXU) val = '%s/signing_cert.pem' % self.signing_dirname self.signing_cert_file_name = val From dfc1ed437954fd9660ca1e89d5f4084abf2f0a8a Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 25 Dec 2012 00:02:54 -0600 Subject: [PATCH 028/120] Use testr instead of nose. Part of blueprint grizzly-testtools Change-Id: I76dee19781eaac21901b5c0258e83a42180c1702 --- test.py | 67 --------------------------------------------------------- 1 file changed, 67 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 77511412..00000000 --- a/test.py +++ /dev/null @@ -1,67 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2012 OpenStack LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# -# Test support for middleware authentication -# - -import os -import sys - - -ROOTDIR = os.path.abspath(os.curdir) - - -def rootdir(*p): - return os.path.join(ROOTDIR, *p) - - -class NoModule(object): - """A mixin class to provide support for unloading/disabling modules.""" - - def __init__(self, *args, **kw): - super(NoModule, self).__init__(*args, **kw) - self._finders = [] - self._cleared_modules = {} - - def tearDown(self): - super(NoModule, self).tearDown() - for finder in self._finders: - sys.meta_path.remove(finder) - sys.modules.update(self._cleared_modules) - - def clear_module(self, module): - cleared_modules = {} - for fullname in sys.modules.keys(): - if fullname == module or fullname.startswith(module + '.'): - cleared_modules[fullname] = sys.modules.pop(fullname) - return cleared_modules - - def disable_module(self, module): - """Ensure ImportError for the specified module.""" - - # Clear 'module' references in sys.modules - self._cleared_modules.update(self.clear_module(module)) - - # Disallow further imports of 'module' - class NoModule(object): - def find_module(self, fullname, path): - if fullname == module or fullname.startswith(module + '.'): - raise ImportError - - finder = NoModule() - self._finders.append(finder) - sys.meta_path.insert(0, finder) From 4e060dd626a730856f2ad0da8111cfb2737d40df Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Fri, 17 May 2013 10:38:25 -0500 Subject: [PATCH 029/120] Default signing_dir to secure temp dir (bug 1181157) Change-Id: I1a29f50b07a60de3d0519bf40074dbea92fa8656 --- auth_token.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index e6cf99f4..befa79e8 100644 --- a/auth_token.py +++ b/auth_token.py @@ -150,6 +150,7 @@ import json import logging import os import stat +import tempfile import time import urllib import webob.exc @@ -211,8 +212,7 @@ opts = [ cfg.StrOpt('cache', default=None), # env key for the swift cache cfg.StrOpt('certfile'), cfg.StrOpt('keyfile'), - cfg.StrOpt('signing_dir', - default=os.path.expanduser('~/keystone-signing')), + cfg.StrOpt('signing_dir'), cfg.ListOpt('memcache_servers'), cfg.IntOpt('token_cache_time', default=300), cfg.IntOpt('revocation_cache_time', default=1), @@ -292,8 +292,10 @@ class AuthProtocol(object): self.cert_file = self._conf_get('certfile') self.key_file = self._conf_get('keyfile') - #signing + # signing self.signing_dirname = self._conf_get('signing_dir') + if self.signing_dirname is None: + self.signing_dirname = tempfile.mkdtemp(prefix='keystone-signing-') self.LOG.info('Using %s as cache directory for signing certificate' % self.signing_dirname) if os.path.exists(self.signing_dirname): From 525a6e187f18547d10917a927266fd7476476b42 Mon Sep 17 00:00:00 2001 From: Adam Young Date: Tue, 28 May 2013 09:50:51 -0400 Subject: [PATCH 030/120] Check Expiry Explicitly checks the expiry on the tokens, and rejects tokens that have expired had to regenerate the sample data for the tokens as they all had been generated with values that are now expired. bug 1179615 Change-Id: Ie06500d446f55fd0ad67ea540c92d8cfc57483f4 --- auth_token.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/auth_token.py b/auth_token.py index befa79e8..69073971 100644 --- a/auth_token.py +++ b/auth_token.py @@ -697,7 +697,8 @@ class AuthProtocol(object): data = json.loads(verified) else: data = self.verify_uuid_token(user_token, retry) - self._cache_put(token_id, data) + expires = self._confirm_token_not_expired(data) + self._cache_put(token_id, data, expires) return data except Exception as e: self.LOG.debug('Token validation failure.', exc_info=True) @@ -931,23 +932,31 @@ class AuthProtocol(object): data_to_store, timeout=self.token_cache_time) - def _cache_put(self, token, data): + def _confirm_token_not_expired(self, data): + if not data: + raise InvalidUserToken('Token authorization failed') + if self._token_is_v2(data): + timestamp = data['access']['token']['expires'] + elif self._token_is_v3(data): + timestamp = data['token']['expires_at'] + else: + raise InvalidUserToken('Token authorization failed') + expires = timeutils.parse_isotime(timestamp).strftime('%s') + if time.time() >= float(expires): + self.LOG.debug('Token expired a %s', timestamp) + raise InvalidUserToken('Token authorization failed') + return expires + + def _cache_put(self, token, data, expires): """ Put token data into the cache. Stores the parsed expire date in cache allowing quick check of token freshness on retrieval. + """ - if self._cache and data: - if self._token_is_v2(data): - timestamp = data['access']['token']['expires'] - elif self._token_is_v3(data): - timestamp = data['token']['expires_at'] - else: - self.LOG.error('invalid token format') - return - expires = timeutils.parse_isotime(timestamp).strftime('%s') - self.LOG.debug('Storing %s token in memcache', token) - self._cache_store(token, data, expires) + if self._cache: + self.LOG.debug('Storing %s token in memcache', token) + self._cache_store(token, data, expires) def _cache_store_invalid(self, token): """Store invalid token in cache.""" From d3b3d4c75cf9cb0209afcb1f46fe96a8b5bc6158 Mon Sep 17 00:00:00 2001 From: "Bryan D. Payne" Date: Fri, 7 Jun 2013 09:34:25 -0700 Subject: [PATCH 031/120] Fix memcache encryption middleware This fixes lp1175367 and lp1175368 by redesigning the memcache crypt middleware to not do dangerous things. It is forward compatible, but will invalidate any existing ephemeral encrypted or signed memcache entries. Change-Id: Ice8724949a48bfad3b8b7c41b5f50a18a9ad9f42 Signed-off-by: Bryan D. Payne --- auth_token.py | 129 ++++++++++++++---------------- memcache_crypt.py | 197 ++++++++++++++++++++++++++++------------------ 2 files changed, 181 insertions(+), 145 deletions(-) diff --git a/auth_token.py b/auth_token.py index 7e3012cb..e50f723c 100644 --- a/auth_token.py +++ b/auth_token.py @@ -222,6 +222,7 @@ opts = [ CONF.register_opts(opts, group='keystone_authtoken') LIST_OF_VERSIONS_TO_ATTEMPT = ['v2.0', 'v3.0'] +CACHE_KEY_TEMPLATE = 'tokens/%s' def will_expire_soon(expiry): @@ -847,91 +848,81 @@ class AuthProtocol(object): env_key = self._header_to_env_var(key) return env.get(env_key, default) - def _protect_cache_value(self, token, data): - """ Encrypt or sign data if necessary. """ - try: - if self._memcache_security_strategy == 'ENCRYPT': - return memcache_crypt.encrypt_data(token, - self._memcache_secret_key, - data) - elif self._memcache_security_strategy == 'MAC': - return memcache_crypt.sign_data(token, data) - else: - return data - except: - msg = 'Failed to encrypt/sign cache data.' - self.LOG.exception(msg) - return data - - def _unprotect_cache_value(self, token, data): - """ Decrypt or verify signed data if necessary. """ - if data is None: - return data - - try: - if self._memcache_security_strategy == 'ENCRYPT': - return memcache_crypt.decrypt_data(token, - self._memcache_secret_key, - data) - elif self._memcache_security_strategy == 'MAC': - return memcache_crypt.verify_signed_data(token, data) - else: - return data - except: - msg = 'Failed to decrypt/verify cache data.' - self.LOG.exception(msg) - # this should have the same effect as data not found in cache - return None - - def _get_cache_key(self, token): - """ Return the cache key. - - Do not use clear token as key if memcache protection is on. - - """ - htoken = token - if self._memcache_security_strategy in ('ENCRYPT', 'MAC'): - derv_token = token + self._memcache_secret_key - htoken = memcache_crypt.hash_data(derv_token) - return 'tokens/%s' % htoken - - def _cache_get(self, token): + def _cache_get(self, token, ignore_expires=False): """Return token information from cache. If token is invalid raise InvalidUserToken return token only if fresh (not expired). """ + if self._cache and token: - key = self._get_cache_key(token) - cached = self._cache.get(key) - cached = self._unprotect_cache_value(token, cached) + if self._memcache_security_strategy is None: + key = CACHE_KEY_TEMPLATE % token + serialized = self._cache.get(key) + else: + keys = memcache_crypt.derive_keys( + token, + self._memcache_secret_key, + self._memcache_security_strategy) + cache_key = CACHE_KEY_TEMPLATE % ( + memcache_crypt.get_cache_key(keys)) + raw_cached = self._cache.get(cache_key) + try: + # unprotect_data will return None if raw_cached is None + serialized = memcache_crypt.unprotect_data(keys, + raw_cached) + except Exception: + msg = 'Failed to decrypt/verify cache data' + self.LOG.exception(msg) + # this should have the same effect as data not + # found in cache + serialized = None + + if serialized is None: + return None + + # Note that 'invalid' and (data, expires) are the only + # valid types of serialized cache entries, so there is not + # a collision with json.loads(serialized) == None. + cached = json.loads(serialized) if cached == 'invalid': self.LOG.debug('Cached Token %s is marked unauthorized', token) raise InvalidUserToken('Token authorization failed') - if cached: - data, expires = cached - if time.time() < float(expires): - self.LOG.debug('Returning cached token %s', token) - return data - else: - self.LOG.debug('Cached Token %s seems expired', token) - def _cache_store(self, token, data, expires=None): - """ Store value into memcache. """ - key = self._get_cache_key(token) - data = self._protect_cache_value(token, data) - data_to_store = data - if expires: - data_to_store = (data, expires) + data, expires = cached + if ignore_expires or time.time() < float(expires): + self.LOG.debug('Returning cached token %s', token) + return data + else: + self.LOG.debug('Cached Token %s seems expired', token) + + def _cache_store(self, token, data): + """ Store value into memcache. + + data may be the string 'invalid' or a tuple like (data, expires) + + """ + serialized_data = json.dumps(data) + if self._memcache_security_strategy is None: + cache_key = CACHE_KEY_TEMPLATE % token + data_to_store = serialized_data + else: + keys = memcache_crypt.derive_keys( + token, + self._memcache_secret_key, + self._memcache_security_strategy) + cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) + data_to_store = memcache_crypt.protect_data(keys, serialized_data) + # we need to special-case set() because of the incompatibility between # Swift MemcacheRing and python-memcached. See # https://bugs.launchpad.net/swift/+bug/1095730 if self._use_keystone_cache: - self._cache.set(key, + self._cache.set(cache_key, data_to_store, time=self.token_cache_time) else: - self._cache.set(key, + self._cache.set(cache_key, data_to_store, timeout=self.token_cache_time) @@ -959,7 +950,7 @@ class AuthProtocol(object): """ if self._cache: self.LOG.debug('Storing %s token in memcache', token) - self._cache_store(token, data, expires) + self._cache_store(token, (data, expires)) def _cache_store_invalid(self, token): """Store invalid token in cache.""" diff --git a/memcache_crypt.py b/memcache_crypt.py index 91e261da..6cadf3ab 100755 --- a/memcache_crypt.py +++ b/memcache_crypt.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010-2012 OpenStack LLC +# Copyright 2010-2013 OpenStack LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,33 +18,34 @@ """ Utilities for memcache encryption and integrity check. -Data is serialized before been encrypted or MACed. Encryption have a -dependency on the pycrypto. If pycrypto is not available, -CryptoUnabailableError will be raised. +Data should be serialized before entering these functions. Encryption +has a dependency on the pycrypto. If pycrypto is not available, +CryptoUnavailableError will be raised. -Encrypted data stored in memcache are prefixed with '{ENCRYPT:AES256}'. - -MACed data stored in memcache are prefixed with '{MAC:SHA1}'. +This module will not be called unless signing or encryption is enabled +in the config. It will always validate signatures, and will decrypt +data if encryption is enabled. It is not valid to mix protection +modes. """ import base64 import functools import hashlib -import json +import hmac +import math import os -# make sure pycrypt is available +# make sure pycrypto is available try: from Crypto.Cipher import AES except ImportError: AES = None - -# prefix marker indicating data is HMACed (signed by a secret key) -MAC_MARKER = '{MAC:SHA1}' -# prefix marker indicating data is encrypted -ENCRYPT_MARKER = '{ENCRYPT:AES256}' +HASH_FUNCTION = hashlib.sha384 +DIGEST_LENGTH = HASH_FUNCTION().digest_size +DIGEST_SPLIT = DIGEST_LENGTH // 3 +DIGEST_LENGTH_B64 = 4 * int(math.ceil(DIGEST_LENGTH / 3.0)) class InvalidMacError(Exception): @@ -81,77 +82,121 @@ def assert_crypto_availability(f): return wrapper -def generate_aes_key(token, secret): - """ Generates and returns a 256 bit AES key, based on sha256 hash. """ - return hashlib.sha256(token + secret).digest() +def constant_time_compare(first, second): + """ Returns True if both string inputs are equal, otherwise False + + This function should take a constant amount of time regardless of + how many characters in the strings match. + + """ + if len(first) != len(second): + return False + result = 0 + for x, y in zip(first, second): + result |= ord(x) ^ ord(y) + return result == 0 -def compute_mac(token, serialized_data): - """ Computes and returns the base64 encoded MAC. """ - return hash_data(serialized_data + token) +def derive_keys(token, secret, strategy): + """ Derives keys for MAC and ENCRYPTION from the user-provided + secret. The resulting keys should be passed to the protect and + unprotect functions. + + As suggested by NIST Special Publication 800-108, this uses the + first 128 bits from the sha384 KDF for the obscured cache key + value, the second 128 bits for the message authentication key and + the remaining 128 bits for the encryption key. + + This approach is faster than computing a separate hmac as the KDF + for each desired key. + """ + digest = hmac.new(secret, token + strategy, HASH_FUNCTION).digest() + return {'CACHE_KEY': digest[:DIGEST_SPLIT], + 'MAC': digest[DIGEST_SPLIT: 2 * DIGEST_SPLIT], + 'ENCRYPTION': digest[2 * DIGEST_SPLIT:], + 'strategy': strategy} -def hash_data(data): - """ Return the base64 encoded SHA1 hash of the data. """ - return base64.b64encode(hashlib.sha1(data).digest()) - - -def sign_data(token, data): - """ MAC the data using SHA1. """ - mac_data = {} - mac_data['serialized_data'] = json.dumps(data) - mac = compute_mac(token, mac_data['serialized_data']) - mac_data['mac'] = mac - md = MAC_MARKER + base64.b64encode(json.dumps(mac_data)) - return md - - -def verify_signed_data(token, data): - """ Verify data integrity by ensuring MAC is valid. """ - if data.startswith(MAC_MARKER): - try: - data = data[len(MAC_MARKER):] - mac_data = json.loads(base64.b64decode(data)) - mac = compute_mac(token, mac_data['serialized_data']) - if mac != mac_data['mac']: - raise InvalidMacError('invalid MAC; expect=%s, actual=%s' % - (mac_data['mac'], mac)) - return json.loads(mac_data['serialized_data']) - except: - raise InvalidMacError('invalid MAC; data appeared to be corrupted') - else: - # doesn't appear to be MACed data - return data +def sign_data(key, data): + """ Sign the data using the defined function and the derived key""" + mac = hmac.new(key, data, HASH_FUNCTION).digest() + return base64.b64encode(mac) @assert_crypto_availability -def encrypt_data(token, secret, data): - """ Encryptes the data with the given secret key. """ +def encrypt_data(key, data): + """ Encrypt the data with the given secret key. + + Padding is n bytes of the value n, where 1 <= n <= blocksize. + """ iv = os.urandom(16) - aes_key = generate_aes_key(token, secret) - cipher = AES.new(aes_key, AES.MODE_CFB, iv) - data = json.dumps(data) - encoded_data = base64.b64encode(iv + cipher.encrypt(data)) - encoded_data = ENCRYPT_MARKER + encoded_data - return encoded_data + cipher = AES.new(key, AES.MODE_CBC, iv) + padding = 16 - len(data) % 16 + return iv + cipher.encrypt(data + chr(padding) * padding) @assert_crypto_availability -def decrypt_data(token, secret, data): +def decrypt_data(key, data): """ Decrypt the data with the given secret key. """ - if data.startswith(ENCRYPT_MARKER): - try: - # encrypted data - encoded_data = data[len(ENCRYPT_MARKER):] - aes_key = generate_aes_key(token, secret) - decoded_data = base64.b64decode(encoded_data) - iv = decoded_data[:16] - encrypted_data = decoded_data[16:] - cipher = AES.new(aes_key, AES.MODE_CFB, iv) - decrypted_data = cipher.decrypt(encrypted_data) - return json.loads(decrypted_data) - except: - raise DecryptError('data appeared to be corrupted') - else: - # doesn't appear to be encrypted data - return data + iv = data[:16] + cipher = AES.new(key, AES.MODE_CBC, iv) + try: + result = cipher.decrypt(data[16:]) + except Exception: + raise DecryptError('Encrypted data appears to be corrupted.') + + # Strip the last n padding bytes where n is the last value in + # the plaintext + padding = ord(result[-1]) + return result[:-1 * padding] + + +def protect_data(keys, data): + """ Given keys and serialized data, returns an appropriately + protected string suitable for storage in the cache. + + """ + if keys['strategy'] == 'ENCRYPT': + data = encrypt_data(keys['ENCRYPTION'], data) + + encoded_data = base64.b64encode(data) + + signature = sign_data(keys['MAC'], encoded_data) + return signature + encoded_data + + +def unprotect_data(keys, signed_data): + """ Given keys and cached string data, verifies the signature, + decrypts if necessary, and returns the original serialized data. + + """ + # cache backends return None when no data is found. We don't mind + # that this particular special value is unsigned. + if signed_data is None: + return None + + # First we calculate the signature + provided_mac = signed_data[:DIGEST_LENGTH_B64] + calculated_mac = sign_data( + keys['MAC'], + signed_data[DIGEST_LENGTH_B64:]) + + # Then verify that it matches the provided value + if not constant_time_compare(provided_mac, calculated_mac): + raise InvalidMacError('Invalid MAC; data appears to be corrupted.') + + data = base64.b64decode(signed_data[DIGEST_LENGTH_B64:]) + + # then if necessary decrypt the data + if keys['strategy'] == 'ENCRYPT': + data = decrypt_data(keys['ENCRYPTION'], data) + + return data + + +def get_cache_key(keys): + """ Given keys generated by derive_keys(), returns a base64 + encoded value suitable for use as a cache key in memcached. + + """ + return base64.b64encode(keys['CACHE_KEY']) From 4810b626658e9f578c55ebd21895360b0167d54c Mon Sep 17 00:00:00 2001 From: xingzhou Date: Tue, 4 Jun 2013 01:44:09 -0400 Subject: [PATCH 032/120] Change memcache config entry name in Keystone to be consistent with Oslo Currently, Keystone-Client is using 'memcache_servers' config option to store the memcache server info, while in OSLO project, it is using 'memcached_servers' to config the same option. It is better to change keystone-client's 'memcache_servers' to 'memcached_servers' to keep consistent between these two projects. Change-Id: I93ca0aa368f95a3ccf6de6984262057e61f75ffe Fixes: Bug 1172793 --- auth_token.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index e50f723c..50844e42 100644 --- a/auth_token.py +++ b/auth_token.py @@ -213,7 +213,7 @@ opts = [ cfg.StrOpt('certfile'), cfg.StrOpt('keyfile'), cfg.StrOpt('signing_dir'), - cfg.ListOpt('memcache_servers'), + cfg.ListOpt('memcached_servers', deprecated_name='memcache_servers'), cfg.IntOpt('token_cache_time', default=300), cfg.IntOpt('revocation_cache_time', default=1), cfg.StrOpt('memcache_security_strategy', default=None), @@ -364,7 +364,8 @@ class AuthProtocol(object): def _init_cache(self, env): cache = self._conf_get('cache') - memcache_servers = self._conf_get('memcache_servers') + memcache_servers = self._conf_get('memcached_servers') + if cache and env.get(cache, None) is not None: # use the cache from the upstream filter self.LOG.info('Using %s memcache for caching token', cache) From a050fe31460fb61b207360eb9680e6595e68ae07 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Thu, 20 Jun 2013 18:24:35 +0200 Subject: [PATCH 033/120] Fix the cache interface to use time= by default. Historically the swift cache conection used the argument timeout= for the cache timeout, but this has been unified with the official python memcache client with time= since grizzly, we still need to handle folsom for a while until this could get removed. Fixes bug 1193032 Change-Id: Ia9d544a0277beb197ed9824d7c1266d12968393d --- auth_token.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/auth_token.py b/auth_token.py index e50f723c..8148d8ad 100644 --- a/auth_token.py +++ b/auth_token.py @@ -331,7 +331,6 @@ class AuthProtocol(object): # Token caching via memcache self._cache = None - self._use_keystone_cache = False self._cache_initialized = False # cache already initialzied? # memcache value treatment, ENCRYPT or MAC self._memcache_security_strategy = \ @@ -372,7 +371,6 @@ class AuthProtocol(object): else: # use Keystone memcache self._cache = memorycache.get_client(memcache_servers) - self._use_keystone_cache = True self._cache_initialized = True def _conf_get(self, name): @@ -914,14 +912,16 @@ class AuthProtocol(object): cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) data_to_store = memcache_crypt.protect_data(keys, serialized_data) - # we need to special-case set() because of the incompatibility between - # Swift MemcacheRing and python-memcached. See - # https://bugs.launchpad.net/swift/+bug/1095730 - if self._use_keystone_cache: + # Historically the swift cache conection used the argument + # timeout= for the cache timeout, but this has been unified + # with the official python memcache client with time= since + # grizzly, we still need to handle folsom for a while until + # this could get removed. + try: self._cache.set(cache_key, data_to_store, time=self.token_cache_time) - else: + except(TypeError): self._cache.set(cache_key, data_to_store, timeout=self.token_cache_time) From 8bd43d639445bc5c4b5c1de3166bcfdc7f43bc5c Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Fri, 21 Jun 2013 19:04:50 +0200 Subject: [PATCH 034/120] Fix and enable H401 Remove leading spaces from doc comments. Change-Id: I75b055c0d64dda478c63839d44158e301900107f --- auth_token.py | 8 ++++---- memcache_crypt.py | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/auth_token.py b/auth_token.py index 89683e62..1b82cfa9 100644 --- a/auth_token.py +++ b/auth_token.py @@ -226,7 +226,7 @@ CACHE_KEY_TEMPLATE = 'tokens/%s' def will_expire_soon(expiry): - """ Determines if expiration is about to occur. + """Determines if expiration is about to occur. :param expiry: a datetime of the expected expiration :returns: boolean : true if expiration is within 30 seconds @@ -382,7 +382,7 @@ class AuthProtocol(object): return CONF.keystone_authtoken[name] def _choose_api_version(self): - """ Determine the api version that we should use.""" + """Determine the api version that we should use.""" # If the configuration specifies an auth_version we will just # assume that is correct and use it. We could, of course, check @@ -896,7 +896,7 @@ class AuthProtocol(object): self.LOG.debug('Cached Token %s seems expired', token) def _cache_store(self, token, data): - """ Store value into memcache. + """Store value into memcache. data may be the string 'invalid' or a tuple like (data, expires) @@ -943,7 +943,7 @@ class AuthProtocol(object): return expires def _cache_put(self, token, data, expires): - """ Put token data into the cache. + """Put token data into the cache. Stores the parsed expire date in cache allowing quick check of token freshness on retrieval. diff --git a/memcache_crypt.py b/memcache_crypt.py index 6cadf3ab..80623338 100755 --- a/memcache_crypt.py +++ b/memcache_crypt.py @@ -49,7 +49,7 @@ DIGEST_LENGTH_B64 = 4 * int(math.ceil(DIGEST_LENGTH / 3.0)) class InvalidMacError(Exception): - """ raise when unable to verify MACed data + """raise when unable to verify MACed data. This usually indicates that data had been expectedly modified in memcache. @@ -58,21 +58,21 @@ class InvalidMacError(Exception): class DecryptError(Exception): - """ raise when unable to decrypt encrypted data + """raise when unable to decrypt encrypted data. """ pass class CryptoUnavailableError(Exception): - """ raise when Python Crypto module is not available + """raise when Python Crypto module is not available. """ pass def assert_crypto_availability(f): - """ Ensure Crypto module is available. """ + """Ensure Crypto module is available.""" @functools.wraps(f) def wrapper(*args, **kwds): @@ -83,7 +83,7 @@ def assert_crypto_availability(f): def constant_time_compare(first, second): - """ Returns True if both string inputs are equal, otherwise False + """Returns True if both string inputs are equal, otherwise False. This function should take a constant amount of time regardless of how many characters in the strings match. @@ -98,7 +98,7 @@ def constant_time_compare(first, second): def derive_keys(token, secret, strategy): - """ Derives keys for MAC and ENCRYPTION from the user-provided + """Derives keys for MAC and ENCRYPTION from the user-provided secret. The resulting keys should be passed to the protect and unprotect functions. @@ -118,14 +118,14 @@ def derive_keys(token, secret, strategy): def sign_data(key, data): - """ Sign the data using the defined function and the derived key""" + """Sign the data using the defined function and the derived key""" mac = hmac.new(key, data, HASH_FUNCTION).digest() return base64.b64encode(mac) @assert_crypto_availability def encrypt_data(key, data): - """ Encrypt the data with the given secret key. + """Encrypt the data with the given secret key. Padding is n bytes of the value n, where 1 <= n <= blocksize. """ @@ -137,7 +137,7 @@ def encrypt_data(key, data): @assert_crypto_availability def decrypt_data(key, data): - """ Decrypt the data with the given secret key. """ + """Decrypt the data with the given secret key.""" iv = data[:16] cipher = AES.new(key, AES.MODE_CBC, iv) try: @@ -152,7 +152,7 @@ def decrypt_data(key, data): def protect_data(keys, data): - """ Given keys and serialized data, returns an appropriately + """Given keys and serialized data, returns an appropriately protected string suitable for storage in the cache. """ @@ -166,7 +166,7 @@ def protect_data(keys, data): def unprotect_data(keys, signed_data): - """ Given keys and cached string data, verifies the signature, + """Given keys and cached string data, verifies the signature, decrypts if necessary, and returns the original serialized data. """ @@ -195,7 +195,7 @@ def unprotect_data(keys, signed_data): def get_cache_key(keys): - """ Given keys generated by derive_keys(), returns a base64 + """Given keys generated by derive_keys(), returns a base64 encoded value suitable for use as a cache key in memcached. """ From 5af2103cc9b4bfd9b282dda9b2f653fd76394e03 Mon Sep 17 00:00:00 2001 From: Donagh McCabe Date: Thu, 4 Jul 2013 11:47:33 +0100 Subject: [PATCH 035/120] Fix auth_token.py bad signing_dir log message Fixes bug 1197755 Change-Id: I16152518c354821af6790a6e459f1d8ec09139a0 --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 1b82cfa9..6537a677 100644 --- a/auth_token.py +++ b/auth_token.py @@ -305,7 +305,7 @@ class AuthProtocol(object): 'unable to access signing_dir %s' % self.signing_dirname) if os.stat(self.signing_dirname).st_uid != os.getuid(): self.LOG.warning( - 'signing_dir is not owned by %s' % os.getlogin()) + 'signing_dir is not owned by %s' % os.getuid()) current_mode = stat.S_IMODE(os.stat(self.signing_dirname).st_mode) if current_mode != stat.S_IRWXU: self.LOG.warning( From 1004914cdef24d2558c1a2882c0aaf2d7c158b69 Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Tue, 2 Jul 2013 10:07:49 +0200 Subject: [PATCH 036/120] Fix and enable gating on H402 Docstring summaries need punctuation. Change-Id: I1b740c13d5fedf9a625ca0807c908f651ee08406 --- memcache_crypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memcache_crypt.py b/memcache_crypt.py index 80623338..f97d08fc 100755 --- a/memcache_crypt.py +++ b/memcache_crypt.py @@ -118,7 +118,7 @@ def derive_keys(token, secret, strategy): def sign_data(key, data): - """Sign the data using the defined function and the derived key""" + """Sign the data using the defined function and the derived key.""" mac = hmac.new(key, data, HASH_FUNCTION).digest() return base64.b64encode(mac) From 0314e77d47a224bebb785f51e8c8eed7ea8b96a3 Mon Sep 17 00:00:00 2001 From: Kun Huang Date: Sun, 14 Jul 2013 18:16:05 +0800 Subject: [PATCH 037/120] rm improper assert syntax assert only raise AssertionError when runing python without '-O', so if we except an Assertionerror in codes, just raise it instead of using assert syntax instead of different result with or without '-O' flag Change-Id: I8c1bc2c6849996f0d3f7a4b791671871d79dd939 --- auth_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 6537a677..46406f2c 100644 --- a/auth_token.py +++ b/auth_token.py @@ -667,8 +667,8 @@ class AuthProtocol(object): try: token = data['access']['token']['id'] expiry = data['access']['token']['expires'] - assert token - assert expiry + if not (token and expiry): + raise AssertionError('invalid token or expire') datetime_expiry = timeutils.parse_isotime(expiry) return (token, timeutils.normalize_time(datetime_expiry)) except (AssertionError, KeyError): From 3f478779bf769ab4b6a79a337f69d4f84a886799 Mon Sep 17 00:00:00 2001 From: Adam Young Date: Mon, 22 Jul 2013 13:50:46 -0400 Subject: [PATCH 038/120] no logging on cms failure Don't log in the keystoneclient.common.cms as there are some errors that are expected. Instead, log in the middleware bug 1189539 Change-Id: I1e80e2ab35e073d9b8d25fd16b31c64c34cd001d --- auth_token.py | 1 + 1 file changed, 1 insertion(+) diff --git a/auth_token.py b/auth_token.py index 6537a677..658cce54 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1047,6 +1047,7 @@ class AuthProtocol(object): if self.cert_file_missing(err.output, self.ca_file_name): self.fetch_ca_cert() continue + self.LOG.warning('Verify error: %s' % err) raise err return output From ee7f54a9bc41ef53f4b48f9e0833c076f40a07e0 Mon Sep 17 00:00:00 2001 From: Pete Zaitcev Date: Fri, 26 Jul 2013 22:21:10 -0600 Subject: [PATCH 039/120] Drop webob from auth_token.py This came up as a packaging problem in Fedora. Now that Swift does not need python-webob RPM, some sites install a proxy node without. We add the proper webob dependency to the python-keystoneclient. Still, in the face of parallel installs, and we end patching setup.py and diverge from upstream, and we do not like it. It may be better just stick to WSGI protocol by small amount of hand-rolled code and drop WebOb altogether. Note that we keep WebOb in the testing code. The fix is purely about the stuff that runs inside a WSGI server. Therefore, requirements.txt continues having no WebOb (apparently, was forgotten before), but test-requirements.txt keeps WebOb in, and this patch changes nothing. Initial version of this patch produced an inexplicable drop in testing coverage and Chmouel put his foot down on it. We were unable to figure out why the drop happened. Most likely coverage simply lied about the previous percentages and getting rid of import webob permitted it a better analysis. So, we add some unrelated tests to bump coverage percentages up. Of course we add a targeted test for the new code too. For that one to work, we change how set_middleware() treats its fake_http argument. This version also adds comments rather than docstrings, because they need to source as context. Change-Id: Iba33b11e77b9910211983aa725c69d3886e1c7a7 --- auth_token.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/auth_token.py b/auth_token.py index 6537a677..cbe00b33 100644 --- a/auth_token.py +++ b/auth_token.py @@ -153,7 +153,6 @@ import stat import tempfile import time import urllib -import webob.exc from keystoneclient.openstack.common import jsonutils from keystoneclient.common import cms @@ -252,6 +251,19 @@ class ConfigurationError(Exception): pass +class MiniResp(object): + def __init__(self, error_message, env, headers=[]): + # The HEAD method is unique: it must never return a body, even if + # it reports an error (RFC-2616 clause 9.4). We relieve callers + # from varying the error responses depending on the method. + if env['REQUEST_METHOD'] == 'HEAD': + self.body = [''] + else: + self.body = [error_message] + self.headers = list(headers) + self.headers.append(('Content-type', 'text/plain')) + + class AuthProtocol(object): """Auth Middleware that handles authenticating client calls.""" @@ -472,8 +484,9 @@ class AuthProtocol(object): except ServiceError as e: self.LOG.critical('Unable to obtain admin token: %s' % e) - resp = webob.exc.HTTPServiceUnavailable() - return resp(env, start_response) + resp = MiniResp('Service unavailable', env) + start_response('503 Service Unavailable', resp.headers) + return resp.body def _remove_auth_headers(self, env): """Remove headers so a user can't fake authentication. @@ -534,8 +547,9 @@ class AuthProtocol(object): """ headers = [('WWW-Authenticate', 'Keystone uri=\'%s\'' % self.auth_uri)] - resp = webob.exc.HTTPUnauthorized('Authentication required', headers) - return resp(env, start_response) + resp = MiniResp('Authentication required', env, headers) + start_response('401 Unauthorized', resp.headers) + return resp.body def get_admin_token(self): """Return admin token, possibly fetching a new one. From 86811e166333c45151bfb81bcdc86775904a7db8 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Thu, 1 Aug 2013 17:09:55 -0500 Subject: [PATCH 040/120] flake8: fix alphabetical imports and enable H306 Change-Id: I0f4fcc9796e8529e7217dc24abe95660633cad33 --- auth_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 4f3fa261..4f9b637e 100644 --- a/auth_token.py +++ b/auth_token.py @@ -155,12 +155,12 @@ import time import urllib import webob.exc -from keystoneclient.openstack.common import jsonutils from keystoneclient.common import cms -from keystoneclient import utils from keystoneclient.middleware import memcache_crypt +from keystoneclient.openstack.common import jsonutils from keystoneclient.openstack.common import memorycache from keystoneclient.openstack.common import timeutils +from keystoneclient import utils CONF = None # to pass gate before oslo-config is deployed everywhere, From f3a4917ea7b98359aa311d3d745bd9684ec6a0ec Mon Sep 17 00:00:00 2001 From: Chuck Short Date: Sun, 4 Aug 2013 01:43:41 +0000 Subject: [PATCH 041/120] python3: Add basic compatibility support Use six.iteritems to replace dictionary.iteritems() on python2 or dictionary.items() on python3. Change-Id: I623009200f3a90985a2c0178673df7d54b36a686 Signed-off-by: Chuck Short --- auth_token.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 3d590022..b19c6a48 100644 --- a/auth_token.py +++ b/auth_token.py @@ -154,6 +154,8 @@ import tempfile import time import urllib +import six + from keystoneclient.common import cms from keystoneclient.middleware import memcache_crypt from keystoneclient.openstack.common import jsonutils @@ -843,7 +845,7 @@ class AuthProtocol(object): def _add_headers(self, env, headers): """Add http headers to environment.""" - for (k, v) in headers.iteritems(): + for (k, v) in six.iteritems(headers): env_key = self._header_to_env_var(k) env[env_key] = v From a23793a64455f8fdff740e190496d3fecaa7b7b1 Mon Sep 17 00:00:00 2001 From: Thomas Goirand Date: Wed, 26 Jun 2013 00:03:37 +0800 Subject: [PATCH 042/120] Adds help in keystone_authtoken config opts The keystone_authtoken config options defined in keystoneclient/middleware/auth_token.py didn't have help strings. Without help, we couldn't generate documented config files via introspection. Fixes Bug1159039 Change-Id: I6d805432edf65db8161d6a6f4916185c4df6bb90 --- auth_token.py | 107 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 22 deletions(-) diff --git a/auth_token.py b/auth_token.py index 3d590022..9e05ad14 100644 --- a/auth_token.py +++ b/auth_token.py @@ -195,28 +195,91 @@ if not CONF: # To use Swift memcache, you must set the 'cache' option to the environment # key where the Swift cache object is stored. opts = [ - cfg.StrOpt('auth_admin_prefix', default=''), - cfg.StrOpt('auth_host', default='127.0.0.1'), - cfg.IntOpt('auth_port', default=35357), - cfg.StrOpt('auth_protocol', default='https'), - cfg.StrOpt('auth_uri', default=None), - cfg.StrOpt('auth_version', default=None), - cfg.BoolOpt('delay_auth_decision', default=False), - cfg.BoolOpt('http_connect_timeout', default=None), - cfg.StrOpt('http_handler', default=None), - cfg.StrOpt('admin_token', secret=True), - cfg.StrOpt('admin_user'), - cfg.StrOpt('admin_password', secret=True), - cfg.StrOpt('admin_tenant_name', default='admin'), - cfg.StrOpt('cache', default=None), # env key for the swift cache - cfg.StrOpt('certfile'), - cfg.StrOpt('keyfile'), - cfg.StrOpt('signing_dir'), - cfg.ListOpt('memcached_servers', deprecated_name='memcache_servers'), - cfg.IntOpt('token_cache_time', default=300), - cfg.IntOpt('revocation_cache_time', default=1), - cfg.StrOpt('memcache_security_strategy', default=None), - cfg.StrOpt('memcache_secret_key', default=None, secret=True) + cfg.StrOpt('auth_admin_prefix', + default='', + help='Prefix to prepend at the begining of the URL'), + cfg.StrOpt('auth_host', + default='127.0.0.1', + help='Host providing the public Identity API endpoint'), + cfg.IntOpt('auth_port', + default=35357, + help='Port of the public Identity API endpoint'), + cfg.StrOpt('auth_protocol', + default='https', + help='Protocol of the public Identity API endpoint' + '(http or https)'), + cfg.StrOpt('auth_uri', + default=None, + help='(optional) Complete public Identity API endpoint;' + ' defaults to auth_protocol://auth_host:auth_port'), + cfg.StrOpt('auth_version', + default=None, + help='API version of the public Identity API endpoint'), + cfg.BoolOpt('delay_auth_decision', + default=False, + help='Do not handle authorization requests within the' + ' middleware, but delegate the authorization decision to' + ' downstream WSGI components'), + cfg.BoolOpt('http_connect_timeout', + default=None, + help='Request timeout value for communicating with Identity' + ' API server.'), + cfg.StrOpt('http_handler', + default=None, + help='Allows to pass in the name of a fake http_handler' + ' callback function used instead of httplib.HTTPConnection or' + ' httplib.HTTPSConnection. Useful for unit testing where' + ' network is not available.'), + cfg.StrOpt('admin_token', + secret=True, + help='Single shared secret with the Keystone configuration' + ' used for bootstrapping a Keystone installation, or otherwise' + ' bypassing the normal authentication process.'), + cfg.StrOpt('admin_user', + help='Keystone account username'), + cfg.StrOpt('admin_password', + secret=True, + help='Keystone account password'), + cfg.StrOpt('admin_tenant_name', + default='admin', + help='Keystone service account tenant name to validate' + ' user tokens'), + cfg.StrOpt('cache', + default=None, + help='Env key for the swift cache'), + cfg.StrOpt('certfile', + help='Required if Keystone server requires client certificate'), + cfg.StrOpt('keyfile', + help='Required if Keystone server requires client certificate'), + cfg.StrOpt('signing_dir', + help='Directory used to cache files related to PKI tokens'), + cfg.ListOpt('memcached_servers', + deprecated_name='memcache_servers', + help='If defined, the memcache server(s) to use for' + ' caching'), + cfg.IntOpt('token_cache_time', + default=300, + help='In order to prevent excessive requests and validations,' + ' the middleware uses an in-memory cache for the tokens the' + ' Keystone API returns. This is only valid if memcache_servers' + ' is defined. Set to -1 to disable caching completely.'), + cfg.IntOpt('revocation_cache_time', + default=1, + help='Value only used for unit testing'), + cfg.StrOpt('memcache_security_strategy', + default=None, + help='(optional) if defined, indicate whether token data' + ' should be authenticated or authenticated and encrypted.' + ' Acceptable values are MAC or ENCRYPT. If MAC, token data is' + ' authenticated (with HMAC) in the cache. If ENCRYPT, token' + ' data is encrypted and authenticated in the cache. If the' + ' value is not one of these options or empty, auth_token will' + ' raise an exception on initialization.'), + cfg.StrOpt('memcache_secret_key', + default=None, + secret=True, + help='(optional, mandatory if memcache_security_strategy is' + ' defined) this string is used for key derivation.') ] CONF.register_opts(opts, group='keystone_authtoken') From 52f6a2f1395bb73be01ecde21b644dfec00049fd Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Thu, 1 Aug 2013 15:53:51 -0500 Subject: [PATCH 043/120] auth_uri (public ep) should not default to auth_* values (admin ep) Fixes bug 1207517 by logging a warning when auth_uri (which should point to the public identity endpoint) falls back on auth_* values (which should point to the admin identity endpoint). Change-Id: I2b051ae10197206f6954672f22e5bff32e3f6c2a --- auth_token.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/auth_token.py b/auth_token.py index 9e05ad14..dc3d17f1 100644 --- a/auth_token.py +++ b/auth_token.py @@ -197,24 +197,26 @@ if not CONF: opts = [ cfg.StrOpt('auth_admin_prefix', default='', - help='Prefix to prepend at the begining of the URL'), + help='Prefix to prepend at the beginning of the path'), cfg.StrOpt('auth_host', default='127.0.0.1', - help='Host providing the public Identity API endpoint'), + help='Host providing the admin Identity API endpoint'), cfg.IntOpt('auth_port', default=35357, - help='Port of the public Identity API endpoint'), + help='Port of the admin Identity API endpoint'), cfg.StrOpt('auth_protocol', default='https', - help='Protocol of the public Identity API endpoint' + help='Protocol of the admin Identity API endpoint' '(http or https)'), cfg.StrOpt('auth_uri', default=None, - help='(optional) Complete public Identity API endpoint;' - ' defaults to auth_protocol://auth_host:auth_port'), + # FIXME(dolph): should be default='http://127.0.0.1:5000/v2.0/', + # or (depending on client support) an unversioned, publicly + # accessible identity endpoint (see bug 1207517) + help='Complete public Identity API endpoint'), cfg.StrOpt('auth_version', default=None, - help='API version of the public Identity API endpoint'), + help='API version of the admin Identity API endpoint'), cfg.BoolOpt('delay_auth_decision', default=False, help='Do not handle authorization requests within the' @@ -360,6 +362,13 @@ class AuthProtocol(object): self.auth_admin_prefix = self._conf_get('auth_admin_prefix') self.auth_uri = self._conf_get('auth_uri') if self.auth_uri is None: + self.LOG.warning( + 'Configuring auth_uri to point to the public identity ' + 'endpoint is required; clients may not be able to ' + 'authenticate against an admin endpoint') + + # FIXME(dolph): drop support for this fallback behavior as + # documented in bug 1207517 self.auth_uri = '%s://%s:%s' % (self.auth_protocol, self.auth_host, self.auth_port) From f6079f16c34bae3bfd6476c5c882bf260a04aa78 Mon Sep 17 00:00:00 2001 From: Matthieu Huin Date: Tue, 6 Aug 2013 16:55:21 +0200 Subject: [PATCH 044/120] Fix a typo in fetch_revocation_list Change-Id: Ida17b6246e6c8dc5ca9c170bd2694e4ff1f24381 --- auth_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index dc3d17f1..0f5cb211 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1203,8 +1203,8 @@ class AuthProtocol(object): return self.fetch_revocation_list(retry=False) if response.status != 200: raise ServiceError('Unable to fetch token revocation list.') - if (not 'signed' in data): - raise ServiceError('Revocation list inmproperly formatted.') + if 'signed' not in data: + raise ServiceError('Revocation list improperly formatted.') return self.cms_verify(data['signed']) def fetch_signing_cert(self): From ee1bb807c9262607f4672d8d1cb1e35581659290 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Wed, 7 Aug 2013 07:22:13 +0200 Subject: [PATCH 045/120] Don't cache tokens as invalid on network errors Closes-Bug: 1172802 Change-Id: Ifc04f565b96d58a7a2f477a216b57af7f8de6a8e --- auth_token.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index 8a65dc69..11949940 100644 --- a/auth_token.py +++ b/auth_token.py @@ -251,6 +251,10 @@ class ConfigurationError(Exception): pass +class NetworkError(Exception): + pass + + class MiniResp(object): def __init__(self, error_message, env, headers=[]): # The HEAD method is unique: it must never return a body, even if @@ -363,6 +367,7 @@ class AuthProtocol(object): self.http_connect_timeout = (http_connect_timeout_cfg and int(http_connect_timeout_cfg)) self.auth_version = None + self.http_request_max_retries = 3 def _assert_valid_memcache_protection_config(self): if self._memcache_security_strategy: @@ -594,9 +599,8 @@ class AuthProtocol(object): """ conn = self._get_http_connection() - RETRIES = 3 + RETRIES = self.http_request_max_retries retry = 0 - while True: try: conn.request(method, path, **kwargs) @@ -606,7 +610,7 @@ class AuthProtocol(object): except Exception as e: if retry == RETRIES: self.LOG.error('HTTP connection exception: %s' % e) - raise ServiceError('Unable to communicate with keystone') + raise NetworkError('Unable to communicate with keystone') # NOTE(vish): sleep 0.5, 1, 2 self.LOG.warn('Retrying on HTTP connection exception: %s' % e) time.sleep(2.0 ** retry / 2) @@ -717,6 +721,10 @@ class AuthProtocol(object): expires = self._confirm_token_not_expired(data) self._cache_put(token_id, data, expires) return data + except NetworkError as e: + self.LOG.debug('Token validation failure.', exc_info=True) + self.LOG.warn("Authorization failed for token %s", user_token) + raise InvalidUserToken('Token authorization failed') except Exception as e: self.LOG.debug('Token validation failure.', exc_info=True) self._cache_store_invalid(user_token) From eec87e62e56175ee1816ded6c4a606e7d06bc161 Mon Sep 17 00:00:00 2001 From: lawrancejing Date: Tue, 13 Aug 2013 21:28:32 +0800 Subject: [PATCH 046/120] Fixes files with wrong bitmode Some modules have bitmode 755. Changed to 644 Change-Id: If8e64c0d27f4d114a4012a4ff34ff08acfec9f06 --- memcache_crypt.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 memcache_crypt.py diff --git a/memcache_crypt.py b/memcache_crypt.py old mode 100755 new mode 100644 From 298ed50ddd09acee4440fbaa4b9dd8c562f6ca1e Mon Sep 17 00:00:00 2001 From: Wu Wenxiang Date: Tue, 30 Jul 2013 23:29:02 +0800 Subject: [PATCH 047/120] Refactor verify signing dir logic Add a function AuthProtocol::verify_signing_dir() in keystoneclient/middleware/auth_token.py. It will be called both in init stage and in fetching signing cert scenario. Fixes bug 1201577 Change-Id: Ice0e8a93ba9d85c20de84cf8da3e9b0f2274b740 --- auth_token.py | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/auth_token.py b/auth_token.py index dc3d17f1..1f67b55d 100644 --- a/auth_token.py +++ b/auth_token.py @@ -383,20 +383,7 @@ class AuthProtocol(object): self.signing_dirname = tempfile.mkdtemp(prefix='keystone-signing-') self.LOG.info('Using %s as cache directory for signing certificate' % self.signing_dirname) - if os.path.exists(self.signing_dirname): - if not os.access(self.signing_dirname, os.W_OK): - raise ConfigurationError( - 'unable to access signing_dir %s' % self.signing_dirname) - if os.stat(self.signing_dirname).st_uid != os.getuid(): - self.LOG.warning( - 'signing_dir is not owned by %s' % os.getuid()) - current_mode = stat.S_IMODE(os.stat(self.signing_dirname).st_mode) - if current_mode != stat.S_IRWXU: - self.LOG.warning( - 'signing_dir mode is %s instead of %s' % - (oct(current_mode), oct(stat.S_IRWXU))) - else: - os.makedirs(self.signing_dirname, stat.S_IRWXU) + self.verify_signing_dir() val = '%s/signing_cert.pem' % self.signing_dirname self.signing_cert_file_name = val @@ -1145,6 +1132,22 @@ class AuthProtocol(object): formatted = cms.token_to_cms(signed_text) return self.cms_verify(formatted) + def verify_signing_dir(self): + if os.path.exists(self.signing_dirname): + if not os.access(self.signing_dirname, os.W_OK): + raise ConfigurationError( + 'unable to access signing_dir %s' % self.signing_dirname) + if os.stat(self.signing_dirname).st_uid != os.getuid(): + self.LOG.warning( + 'signing_dir is not owned by %s' % os.getuid()) + current_mode = stat.S_IMODE(os.stat(self.signing_dirname).st_mode) + if current_mode != stat.S_IRWXU: + self.LOG.warning( + 'signing_dir mode is %s instead of %s' % + (oct(current_mode), oct(stat.S_IRWXU))) + else: + os.makedirs(self.signing_dirname, stat.S_IRWXU) + @property def token_revocation_list_fetched_time(self): if not self._token_revocation_list_fetched_time: @@ -1210,11 +1213,19 @@ class AuthProtocol(object): def fetch_signing_cert(self): response, data = self._http_request('GET', '/v2.0/certificates/signing') - try: - #todo check response + + def write_cert_file(data): certfile = open(self.signing_cert_file_name, 'w') certfile.write(data) certfile.close() + + try: + #todo check response + try: + write_cert_file(data) + except IOError: + self.verify_signing_dir() + write_cert_file(data) except (AssertionError, KeyError): self.LOG.warn( "Unexpected response from keystone service: %s", data) From ce707441b830eb990a35542a2162f9a3115853a1 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Mon, 12 Aug 2013 15:01:22 -0400 Subject: [PATCH 048/120] Move all opens in auth_token to be in context This commit switches the use of open() to be in a with context. This will prevent leaking file descriptors because using a context provides an inherent close(). Prior to this commit it was possible that an exception or error during the write() call could prevent close() from running. Change-Id: Ib13fe651d41f0eafea23a1c96c6c64d405de7e49 --- auth_token.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/auth_token.py b/auth_token.py index 5e1e1e61..1af26feb 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1217,9 +1217,8 @@ class AuthProtocol(object): '/v2.0/certificates/signing') def write_cert_file(data): - certfile = open(self.signing_cert_file_name, 'w') - certfile.write(data) - certfile.close() + with open(self.signing_cert_file_name, 'w') as certfile: + certfile.write(data) try: #todo check response @@ -1238,9 +1237,8 @@ class AuthProtocol(object): '/v2.0/certificates/ca') try: #todo check response - certfile = open(self.ca_file_name, 'w') - certfile.write(data) - certfile.close() + with open(self.ca_file_name, 'w') as certfile: + certfile.write(data) except (AssertionError, KeyError): self.LOG.warn( "Unexpected response from keystone service: %s", data) From 6eadfb3c23bb4a51ac4c66e5e1b111b779cbbc13 Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Tue, 13 Aug 2013 13:25:39 +1000 Subject: [PATCH 049/120] Make auth_token middleware fetching respect prefix In auth_token middleware auth_prefix is only added to requests that go through the _json_request method. This doesn't include the two certificate fetching functions. Manually add the auth_prefix to both those requests. Closes-Bug: #1211615 Change-Id: I25d1b401598c9a443ddef0fc3259ba859aee8c76 --- auth_token.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/auth_token.py b/auth_token.py index e2628b62..f2a0991e 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1221,8 +1221,9 @@ class AuthProtocol(object): return self.cms_verify(data['signed']) def fetch_signing_cert(self): - response, data = self._http_request('GET', - '/v2.0/certificates/signing') + path = self.auth_admin_prefix.rstrip('/') + path += '/v2.0/certificates/signing' + response, data = self._http_request('GET', path) def write_cert_file(data): certfile = open(self.signing_cert_file_name, 'w') @@ -1242,8 +1243,9 @@ class AuthProtocol(object): raise ServiceError('invalid json response') def fetch_ca_cert(self): - response, data = self._http_request('GET', - '/v2.0/certificates/ca') + path = self.auth_admin_prefix.rstrip('/') + '/v2.0/certificates/ca' + response, data = self._http_request('GET', path) + try: #todo check response certfile = open(self.ca_file_name, 'w') From 94af21c8bdcf9566175b8d90e5115fd43b02958d Mon Sep 17 00:00:00 2001 From: Kieran Spear Date: Tue, 30 Jul 2013 14:24:38 +1000 Subject: [PATCH 050/120] Use hashed token for invalid PKI token cache key Invalid PKI tokens are cached in memcache using the entire token as key. This triggers the familiar memcache key length error since a PKI token is much longer than 250 characters. This change hashes the token before using it as a key, and also changes instances of "token" to "token_id" where appropriate so it's clear that we're passing around a hashed token rather than the token data itself. Change-Id: I765b209578d60266da706094f61d8d9b15cfb6de Closes-bug: 1206347 --- auth_token.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/auth_token.py b/auth_token.py index e2628b62..e606b9c5 100644 --- a/auth_token.py +++ b/auth_token.py @@ -769,6 +769,8 @@ class AuthProtocol(object): :no longer raises ServiceError since it no longer makes RPC """ + token_id = None + try: token_id = cms.cms_hash_token(user_token) cached = self._cache_get(token_id) @@ -784,12 +786,13 @@ class AuthProtocol(object): return data except NetworkError as e: self.LOG.debug('Token validation failure.', exc_info=True) - self.LOG.warn("Authorization failed for token %s", user_token) + self.LOG.warn("Authorization failed for token %s", token_id) raise InvalidUserToken('Token authorization failed') except Exception as e: self.LOG.debug('Token validation failure.', exc_info=True) - self._cache_store_invalid(user_token) - self.LOG.warn("Authorization failed for token %s", user_token) + if token_id: + self._cache_store_invalid(token_id) + self.LOG.warn("Authorization failed for token %s", token_id) raise InvalidUserToken('Token authorization failed') def _token_is_v2(self, token_info): @@ -930,20 +933,20 @@ class AuthProtocol(object): env_key = self._header_to_env_var(key) return env.get(env_key, default) - def _cache_get(self, token, ignore_expires=False): + def _cache_get(self, token_id, ignore_expires=False): """Return token information from cache. If token is invalid raise InvalidUserToken return token only if fresh (not expired). """ - if self._cache and token: + if self._cache and token_id: if self._memcache_security_strategy is None: - key = CACHE_KEY_TEMPLATE % token + key = CACHE_KEY_TEMPLATE % token_id serialized = self._cache.get(key) else: keys = memcache_crypt.derive_keys( - token, + token_id, self._memcache_secret_key, self._memcache_security_strategy) cache_key = CACHE_KEY_TEMPLATE % ( @@ -968,17 +971,18 @@ class AuthProtocol(object): # a collision with json.loads(serialized) == None. cached = json.loads(serialized) if cached == 'invalid': - self.LOG.debug('Cached Token %s is marked unauthorized', token) + self.LOG.debug('Cached Token %s is marked unauthorized', + token_id) raise InvalidUserToken('Token authorization failed') data, expires = cached if ignore_expires or time.time() < float(expires): - self.LOG.debug('Returning cached token %s', token) + self.LOG.debug('Returning cached token %s', token_id) return data else: - self.LOG.debug('Cached Token %s seems expired', token) + self.LOG.debug('Cached Token %s seems expired', token_id) - def _cache_store(self, token, data): + def _cache_store(self, token_id, data): """Store value into memcache. data may be the string 'invalid' or a tuple like (data, expires) @@ -986,11 +990,11 @@ class AuthProtocol(object): """ serialized_data = json.dumps(data) if self._memcache_security_strategy is None: - cache_key = CACHE_KEY_TEMPLATE % token + cache_key = CACHE_KEY_TEMPLATE % token_id data_to_store = serialized_data else: keys = memcache_crypt.derive_keys( - token, + token_id, self._memcache_secret_key, self._memcache_security_strategy) cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) @@ -1025,7 +1029,7 @@ class AuthProtocol(object): raise InvalidUserToken('Token authorization failed') return expires - def _cache_put(self, token, data, expires): + def _cache_put(self, token_id, data, expires): """Put token data into the cache. Stores the parsed expire date in cache allowing @@ -1033,15 +1037,15 @@ class AuthProtocol(object): """ if self._cache: - self.LOG.debug('Storing %s token in memcache', token) - self._cache_store(token, (data, expires)) + self.LOG.debug('Storing %s token in memcache', token_id) + self._cache_store(token_id, (data, expires)) - def _cache_store_invalid(self, token): + def _cache_store_invalid(self, token_id): """Store invalid token in cache.""" if self._cache: self.LOG.debug( - 'Marking token %s as unauthorized in memcache', token) - self._cache_store(token, 'invalid') + 'Marking token %s as unauthorized in memcache', token_id) + self._cache_store(token_id, 'invalid') def cert_file_missing(self, proc_output, file_name): return (file_name in proc_output and not os.path.exists(file_name)) From ce27d62b0353aeb65189d3684ec5c68b941fd4c0 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Wed, 21 Aug 2013 09:36:27 +0200 Subject: [PATCH 051/120] Allow configure the number of http retries Allow setting the maximum number of times we want to retry to http_connect when communicating with the Identity service. Closes-Bug: 1209194 Change-Id: I56bff55dc808a1188d42f1643a8018a8d49012c6 --- auth_token.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index f2a0991e..f58de4b6 100644 --- a/auth_token.py +++ b/auth_token.py @@ -228,6 +228,10 @@ opts = [ default=None, help='Request timeout value for communicating with Identity' ' API server.'), + cfg.IntOpt('http_request_max_retries', + default=3, + help='How many times are we trying to reconnect when' + ' communicating with Identity API Server.'), cfg.StrOpt('http_handler', default=None, help='Allows to pass in the name of a fake http_handler' @@ -428,7 +432,8 @@ class AuthProtocol(object): self.http_connect_timeout = (http_connect_timeout_cfg and int(http_connect_timeout_cfg)) self.auth_version = None - self.http_request_max_retries = 3 + self.http_request_max_retries = \ + self._conf_get('http_request_max_retries') def _assert_valid_memcache_protection_config(self): if self._memcache_security_strategy: From eed68d621830e3d238a95fcb887db959bb5f897e Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Fri, 16 Aug 2013 11:45:44 +1000 Subject: [PATCH 052/120] Use OSLO jsonutils instead of json module For most things there is very little difference, but as we have a jsonutils module we should probably be using it. Change-Id: I406ea81bb56ad90cc9ff9b8b58b0d35b694dc802 --- auth_token.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/auth_token.py b/auth_token.py index 6fc1ffab..126d156b 100644 --- a/auth_token.py +++ b/auth_token.py @@ -146,7 +146,6 @@ keystone.token_info import datetime import httplib -import json import logging import os import stat @@ -781,7 +780,7 @@ class AuthProtocol(object): return cached if cms.is_ans1_token(user_token): verified = self.verify_signed_token(user_token) - data = json.loads(verified) + data = jsonutils.loads(verified) else: data = self.verify_uuid_token(user_token, retry) expires = self._confirm_token_not_expired(data) @@ -970,8 +969,8 @@ class AuthProtocol(object): # Note that 'invalid' and (data, expires) are the only # valid types of serialized cache entries, so there is not - # a collision with json.loads(serialized) == None. - cached = json.loads(serialized) + # a collision with jsonutils.loads(serialized) == None. + cached = jsonutils.loads(serialized) if cached == 'invalid': self.LOG.debug('Cached Token %s is marked unauthorized', token) raise InvalidUserToken('Token authorization failed') @@ -989,7 +988,7 @@ class AuthProtocol(object): data may be the string 'invalid' or a tuple like (data, expires) """ - serialized_data = json.dumps(data) + serialized_data = jsonutils.dumps(data) if self._memcache_security_strategy is None: cache_key = CACHE_KEY_TEMPLATE % token data_to_store = serialized_data From 150ce4e3e3f41cf97d94063824a397ab7819e85b Mon Sep 17 00:00:00 2001 From: Morgan Fainberg Date: Thu, 29 Aug 2013 15:07:50 -0700 Subject: [PATCH 053/120] Fix and enable gating on F841 F841 local variable is assigned to but never used Change-Id: I4b54489386deb655821192b4ec1e9c0ea596a9b7 --- auth_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 4de1cfbd..aad10261 100644 --- a/auth_token.py +++ b/auth_token.py @@ -789,11 +789,11 @@ class AuthProtocol(object): expires = self._confirm_token_not_expired(data) self._cache_put(token_id, data, expires) return data - except NetworkError as e: + except NetworkError: self.LOG.debug('Token validation failure.', exc_info=True) self.LOG.warn("Authorization failed for token %s", token_id) raise InvalidUserToken('Token authorization failed') - except Exception as e: + except Exception: self.LOG.debug('Token validation failure.', exc_info=True) if token_id: self._cache_store_invalid(token_id) From e685e77a13bb477fd8d6161bc9b198b14855153f Mon Sep 17 00:00:00 2001 From: AmalaBasha Date: Tue, 3 Sep 2013 15:51:59 +0530 Subject: [PATCH 054/120] Log user info in auth_token middleware Add logging for user information (like user name, tenant_id, roles) in the auth_token middleware. This would make tracking down issues much easier. Change-Id: Ife4ce29d2f8e1a338a025dda4afbd7b563f6b8c1 Implements: blueprint user-info-logging-in-auth-token-middleware --- auth_token.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/auth_token.py b/auth_token.py index 6dc7c931..da32a508 100644 --- a/auth_token.py +++ b/auth_token.py @@ -900,6 +900,9 @@ class AuthProtocol(object): 'X-Role': roles, } + self.LOG.debug("Received request from user: %s with project_id : %s" + " and roles: %s ", user_id, project_id, roles) + try: catalog = catalog_root[catalog_key] rval['X-Service-Catalog'] = jsonutils.dumps(catalog) From c7746bcf368450872f88a30c71f4bd11fa3bd060 Mon Sep 17 00:00:00 2001 From: Dazhao Date: Wed, 21 Aug 2013 11:43:36 +0800 Subject: [PATCH 055/120] Support client generate literal ipv6 auth_uri base on auth_host This patch is for fix bug 1208784. Support keystone client generate literal ipv6 auth_uri via provided auth_host, when the auth_uri is None. Fixes bug 1208784 Change-Id: I226881a35a74ef668d4cd1c6829a64c94ff185d9 --- auth_token.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 6dc7c931..769b61b6 100644 --- a/auth_token.py +++ b/auth_token.py @@ -153,6 +153,7 @@ import tempfile import time import urllib +import netaddr import six from keystoneclient.common import cms @@ -375,11 +376,16 @@ class AuthProtocol(object): 'Configuring auth_uri to point to the public identity ' 'endpoint is required; clients may not be able to ' 'authenticate against an admin endpoint') - + host = self.auth_host + if netaddr.valid_ipv6(host): + # Note(dzyu) it is an IPv6 address, so it needs to be wrapped + # with '[]' to generate a valid IPv6 URL, based on + # http://www.ietf.org/rfc/rfc2732.txt + host = '[%s]' % host # FIXME(dolph): drop support for this fallback behavior as # documented in bug 1207517 self.auth_uri = '%s://%s:%s' % (self.auth_protocol, - self.auth_host, + host, self.auth_port) # SSL From 444efc90088f9213f3198bf5644604e2e8f7506a Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Mon, 12 Aug 2013 13:12:27 +1000 Subject: [PATCH 056/120] Replace HttpConnection in auth_token with Requests Requests is becoming the standard way of doing http communication, it also vastly simplifies adding other authentication mechanisms. Use it in the auth_token middleware. This adds the ability to specify a CA file that will be used to verify a HTTPS connections or insecure to specifically ignore HTTPS validation. SecurityImpact DocImpact Partial-Bug: #1188189 Change-Id: Iae94329e7abd105bf95224d28f39f4b746b9eb70 --- auth_token.py | 129 ++++++++++++++++++++++++-------------------------- 1 file changed, 61 insertions(+), 68 deletions(-) diff --git a/auth_token.py b/auth_token.py index 769b61b6..1dc0a7b1 100644 --- a/auth_token.py +++ b/auth_token.py @@ -145,9 +145,9 @@ keystone.token_info """ import datetime -import httplib import logging import os +import requests import stat import tempfile import time @@ -259,6 +259,10 @@ opts = [ help='Required if Keystone server requires client certificate'), cfg.StrOpt('keyfile', help='Required if Keystone server requires client certificate'), + cfg.StrOpt('cafile', default=None, + help='A PEM encoded Certificate Authority to use when ' + 'verifying HTTPs connections. Defaults to system CAs.'), + cfg.BoolOpt('insecure', default=False, help='Verify HTTPS connections.'), cfg.StrOpt('signing_dir', help='Directory used to cache files related to PKI tokens'), cfg.ListOpt('memcached_servers', @@ -354,43 +358,35 @@ class AuthProtocol(object): (True, 'true', 't', '1', 'on', 'yes', 'y')) # where to find the auth service (we use this to validate tokens) - self.auth_host = self._conf_get('auth_host') - self.auth_port = int(self._conf_get('auth_port')) - self.auth_protocol = self._conf_get('auth_protocol') - if not self._conf_get('http_handler'): - if self.auth_protocol == 'http': - self.http_client_class = httplib.HTTPConnection - else: - self.http_client_class = httplib.HTTPSConnection - else: - # Really only used for unit testing, since we need to - # have a fake handler set up before we issue an http - # request to get the list of versions supported by the - # server at the end of this initialization - self.http_client_class = self._conf_get('http_handler') - + auth_host = self._conf_get('auth_host') + auth_port = int(self._conf_get('auth_port')) + auth_protocol = self._conf_get('auth_protocol') self.auth_admin_prefix = self._conf_get('auth_admin_prefix') self.auth_uri = self._conf_get('auth_uri') + + if netaddr.valid_ipv6(auth_host): + # Note(dzyu) it is an IPv6 address, so it needs to be wrapped + # with '[]' to generate a valid IPv6 URL, based on + # http://www.ietf.org/rfc/rfc2732.txt + auth_host = '[%s]' % auth_host + + self.request_uri = '%s://%s:%s' % (auth_protocol, auth_host, auth_port) + if self.auth_uri is None: self.LOG.warning( 'Configuring auth_uri to point to the public identity ' 'endpoint is required; clients may not be able to ' 'authenticate against an admin endpoint') - host = self.auth_host - if netaddr.valid_ipv6(host): - # Note(dzyu) it is an IPv6 address, so it needs to be wrapped - # with '[]' to generate a valid IPv6 URL, based on - # http://www.ietf.org/rfc/rfc2732.txt - host = '[%s]' % host + # FIXME(dolph): drop support for this fallback behavior as # documented in bug 1207517 - self.auth_uri = '%s://%s:%s' % (self.auth_protocol, - host, - self.auth_port) + self.auth_uri = self.request_uri # SSL self.cert_file = self._conf_get('certfile') self.key_file = self._conf_get('keyfile') + self.ssl_ca_file = self._conf_get('cafile') + self.ssl_insecure = self._conf_get('insecure') # signing self.signing_dirname = self._conf_get('signing_dir') @@ -403,7 +399,7 @@ class AuthProtocol(object): val = '%s/signing_cert.pem' % self.signing_dirname self.signing_cert_file_name = val val = '%s/cacert.pem' % self.signing_dirname - self.ca_file_name = val + self.signing_ca_file_name = val val = '%s/revoked.pem' % self.signing_dirname self.revoked_file_name = val @@ -505,12 +501,12 @@ class AuthProtocol(object): def _get_supported_versions(self): versions = [] response, data = self._json_request('GET', '/') - if response.status == 501: + if response.status_code == 501: self.LOG.warning("Old keystone installation found...assuming v2.0") versions.append("v2.0") - elif response.status != 300: + elif response.status_code != 300: self.LOG.error('Unable to get version info from keystone: %s' % - response.status) + response.status_code) raise ServiceError('Unable to get version info from keystone') else: try: @@ -648,17 +644,6 @@ class AuthProtocol(object): return self.admin_token - def _get_http_connection(self): - if self.auth_protocol == 'http': - return self.http_client_class(self.auth_host, self.auth_port, - timeout=self.http_connect_timeout) - else: - return self.http_client_class(self.auth_host, - self.auth_port, - self.key_file, - self.cert_file, - timeout=self.http_connect_timeout) - def _http_request(self, method, path, **kwargs): """HTTP request helper used to make unspecified content type requests. @@ -668,28 +653,35 @@ class AuthProtocol(object): :raise ServerError when unable to communicate with keystone """ - conn = self._get_http_connection() + url = "%s/%s" % (self.request_uri, path.lstrip('/')) + + kwargs.setdefault('timeout', self.http_connect_timeout) + if self.cert_file and self.key_file: + kwargs['cert'] = (self.cert_file, self.key_file) + elif self.cert_file or self.key_file: + self.LOG.warn('Cannot use only a cert or key file. ' + 'Please provide both. Ignoring.') + + kwargs['verify'] = self.ssl_ca_file or True + if self.ssl_insecure: + kwargs['verify'] = False RETRIES = self.http_request_max_retries retry = 0 while True: try: - conn.request(method, path, **kwargs) - response = conn.getresponse() - body = response.read() + response = requests.request(method, url, **kwargs) break except Exception as e: - if retry == RETRIES: - self.LOG.error('HTTP connection exception: %s' % e) + if retry >= RETRIES: + self.LOG.error('HTTP connection exception: %s', e) raise NetworkError('Unable to communicate with keystone') # NOTE(vish): sleep 0.5, 1, 2 self.LOG.warn('Retrying on HTTP connection exception: %s' % e) time.sleep(2.0 ** retry / 2) retry += 1 - finally: - conn.close() - return response, body + return response def _json_request(self, method, path, body=None, additional_headers=None): """HTTP request helper used to make json requests. @@ -714,14 +706,14 @@ class AuthProtocol(object): kwargs['headers'].update(additional_headers) if body: - kwargs['body'] = jsonutils.dumps(body) + kwargs['data'] = jsonutils.dumps(body) path = self.auth_admin_prefix + path - response, body = self._http_request(method, path, **kwargs) + response = self._http_request(method, path, **kwargs) try: - data = jsonutils.loads(body) + data = jsonutils.loads(response.text) except ValueError: self.LOG.debug('Keystone did not return json-encoded body') data = {} @@ -1090,18 +1082,18 @@ class AuthProtocol(object): '/v2.0/tokens/%s' % safe_quote(user_token), additional_headers=headers) - if response.status == 200: + if response.status_code == 200: return data - if response.status == 404: + if response.status_code == 404: self.LOG.warn("Authorization failed for token %s", user_token) raise InvalidUserToken('Token authorization failed') - if response.status == 401: + if response.status_code == 401: self.LOG.info( 'Keystone rejected admin token %s, resetting', headers) self.admin_token = None else: self.LOG.error('Bad response code while validating token: %s' % - response.status) + response.status_code) if retry: self.LOG.info('Retrying validation') return self._validate_user_token(user_token, False) @@ -1135,13 +1127,14 @@ class AuthProtocol(object): while True: try: output = cms.cms_verify(data, self.signing_cert_file_name, - self.ca_file_name) + self.signing_ca_file_name) except cms.subprocess.CalledProcessError as err: if self.cert_file_missing(err.output, self.signing_cert_file_name): self.fetch_signing_cert() continue - if self.cert_file_missing(err.output, self.ca_file_name): + if self.cert_file_missing(err.output, + self.signing_ca_file_name): self.fetch_ca_cert() continue self.LOG.warning('Verify error: %s' % err) @@ -1221,14 +1214,14 @@ class AuthProtocol(object): headers = {'X-Auth-Token': self.get_admin_token()} response, data = self._json_request('GET', '/v2.0/tokens/revoked', additional_headers=headers) - if response.status == 401: + if response.status_code == 401: if retry: self.LOG.info( 'Keystone rejected admin token %s, resetting admin token', headers) self.admin_token = None return self.fetch_revocation_list(retry=False) - if response.status != 200: + if response.status_code != 200: raise ServiceError('Unable to fetch token revocation list.') if 'signed' not in data: raise ServiceError('Revocation list improperly formatted.') @@ -1237,7 +1230,7 @@ class AuthProtocol(object): def fetch_signing_cert(self): path = self.auth_admin_prefix.rstrip('/') path += '/v2.0/certificates/signing' - response, data = self._http_request('GET', path) + response = self._http_request('GET', path) def write_cert_file(data): with open(self.signing_cert_file_name, 'w') as certfile: @@ -1246,26 +1239,26 @@ class AuthProtocol(object): try: #todo check response try: - write_cert_file(data) + write_cert_file(response.text) except IOError: self.verify_signing_dir() - write_cert_file(data) + write_cert_file(response.text) except (AssertionError, KeyError): self.LOG.warn( - "Unexpected response from keystone service: %s", data) + "Unexpected response from keystone service: %s", response.text) raise ServiceError('invalid json response') def fetch_ca_cert(self): path = self.auth_admin_prefix.rstrip('/') + '/v2.0/certificates/ca' - response, data = self._http_request('GET', path) + response = self._http_request('GET', path) try: #todo check response - with open(self.ca_file_name, 'w') as certfile: - certfile.write(data) + with open(self.signing_ca_file_name, 'w') as certfile: + certfile.write(response.text) except (AssertionError, KeyError): self.LOG.warn( - "Unexpected response from keystone service: %s", data) + "Unexpected response from keystone service: %s", response.text) raise ServiceError('invalid json response') From f02eec357442bf6f604d62278ae4eb4b5084cfb4 Mon Sep 17 00:00:00 2001 From: Bryan Davidson Date: Fri, 30 Aug 2013 12:31:12 -0400 Subject: [PATCH 057/120] Refactor for testability of an upcoming change confirm_token_not_expired() in keystoneclient/middleware/auth_token.py has been moved out of the class to make it a function and be more testable. Currently, there is no need to keep it within the class. An upcoming commit makes fixes that rely on this refactor to be tested. Change-Id: I8460a2ee663dec8be0f339735208779a3b988040 --- auth_token.py | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/auth_token.py b/auth_token.py index 1dc0a7b1..616c18de 100644 --- a/auth_token.py +++ b/auth_token.py @@ -309,6 +309,29 @@ def will_expire_soon(expiry): return expiry < soon +def _token_is_v2(token_info): + return ('access' in token_info) + + +def _token_is_v3(token_info): + return ('token' in token_info) + + +def confirm_token_not_expired(data): + if not data: + raise InvalidUserToken('Token authorization failed') + if _token_is_v2(data): + timestamp = data['access']['token']['expires'] + elif _token_is_v3(data): + timestamp = data['token']['expires_at'] + else: + raise InvalidUserToken('Token authorization failed') + expires = timeutils.parse_isotime(timestamp).strftime('%s') + if time.time() >= float(expires): + raise InvalidUserToken('Token authorization failed') + return expires + + def safe_quote(s): """URL-encode strings that are not already URL-encoded.""" return urllib.quote(s) if s == urllib.unquote(s) else s @@ -783,7 +806,7 @@ class AuthProtocol(object): data = jsonutils.loads(verified) else: data = self.verify_uuid_token(user_token, retry) - expires = self._confirm_token_not_expired(data) + expires = confirm_token_not_expired(data) self._cache_put(token_id, data, expires) return data except NetworkError: @@ -797,12 +820,6 @@ class AuthProtocol(object): self.LOG.warn("Authorization failed for token %s", token_id) raise InvalidUserToken('Token authorization failed') - def _token_is_v2(self, token_info): - return ('access' in token_info) - - def _token_is_v3(self, token_info): - return ('token' in token_info) - def _build_user_headers(self, token_info): """Convert token object into headers. @@ -846,7 +863,7 @@ class AuthProtocol(object): project_domain_id = None project_domain_name = None - if self._token_is_v2(token_info): + if _token_is_v2(token_info): user = token_info['access']['user'] token = token_info['access']['token'] roles = ','.join([role['name'] for role in user.get('roles', [])]) @@ -1016,21 +1033,6 @@ class AuthProtocol(object): data_to_store, timeout=self.token_cache_time) - def _confirm_token_not_expired(self, data): - if not data: - raise InvalidUserToken('Token authorization failed') - if self._token_is_v2(data): - timestamp = data['access']['token']['expires'] - elif self._token_is_v3(data): - timestamp = data['token']['expires_at'] - else: - raise InvalidUserToken('Token authorization failed') - expires = timeutils.parse_isotime(timestamp).strftime('%s') - if time.time() >= float(expires): - self.LOG.debug('Token expired a %s', timestamp) - raise InvalidUserToken('Token authorization failed') - return expires - def _cache_put(self, token_id, data, expires): """Put token data into the cache. From cd81cbd476da0c11cf9afd28f1df50b86a57120f Mon Sep 17 00:00:00 2001 From: Kui Shi Date: Fri, 4 Oct 2013 16:54:03 +0800 Subject: [PATCH 058/120] Fix H202 assertRaises Exception Align the hacking version between test-requirement and global requirement. The change of H202 detection from 0.6 to 0.7 in hacking is: - if logical_line.startswith("self.assertRaises(Exception)"): + if logical_line.startswith("self.assertRaises(Exception,"): then more cases are detected by this change. Fix the exposed H202 error. There is a special test case: tests/v3/test_endpoints.py:test_update_invalid_interface ref = self.new_ref(interface=uuid.uuid4().hex) this line can not generate proper parameter for self.manager.update, add a parameter "endpoint" for it, according to the definition in keystoneclient/v3/endpoints.py:EndpointManager.update. Otherwise, there will be following error after changing the Exception to exceptions.ValidationError: TypeError: update() takes at least 2 arguments (6 given) Fixes Bug #1220008 Change-Id: I8f7ed7a6eebf8576a6db5fecd86b9d19a15c8d60 --- auth_token.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auth_token.py b/auth_token.py index 740dff28..48b9f465 100644 --- a/auth_token.py +++ b/auth_token.py @@ -439,11 +439,12 @@ class AuthProtocol(object): def _assert_valid_memcache_protection_config(self): if self._memcache_security_strategy: if self._memcache_security_strategy not in ('MAC', 'ENCRYPT'): - raise Exception('memcache_security_strategy must be ' - 'ENCRYPT or MAC') + raise ConfigurationError('memcache_security_strategy must be ' + 'ENCRYPT or MAC') if not self._memcache_secret_key: - raise Exception('mecmache_secret_key must be defined when ' - 'a memcache_security_strategy is defined') + raise ConfigurationError('mecmache_secret_key must be defined ' + 'when a memcache_security_strategy ' + 'is defined') def _init_cache(self, env): cache = self._conf_get('cache') From 909042532f011a5884d879103580665d58a820b1 Mon Sep 17 00:00:00 2001 From: Bryan Davidson Date: Fri, 30 Aug 2013 13:38:37 -0400 Subject: [PATCH 059/120] Normalize datetimes to account for tz This patch makes sure that datetimes in the auth_token middleware are normalized to account for timezone offsets. Some of the old tests were changed to ensure that the expires string stored in the cache is in ISO 8601 format and not a random float. Fixes bug 1195924 Change-Id: I5917ab728193cd2aa8784c4860a96cdc17f3d43f --- auth_token.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/auth_token.py b/auth_token.py index cd89cf15..dde5b027 100644 --- a/auth_token.py +++ b/auth_token.py @@ -326,10 +326,12 @@ def confirm_token_not_expired(data): timestamp = data['token']['expires_at'] else: raise InvalidUserToken('Token authorization failed') - expires = timeutils.parse_isotime(timestamp).strftime('%s') - if time.time() >= float(expires): + expires = timeutils.parse_isotime(timestamp) + expires = timeutils.normalize_time(expires) + utcnow = timeutils.utcnow() + if utcnow >= expires: raise InvalidUserToken('Token authorization failed') - return expires + return timeutils.isotime(at=expires, subsecond=True) def safe_quote(s): @@ -998,7 +1000,18 @@ class AuthProtocol(object): raise InvalidUserToken('Token authorization failed') data, expires = cached - if ignore_expires or time.time() < float(expires): + + try: + expires = timeutils.parse_isotime(expires) + except ValueError: + # Gracefully handle upgrade of expiration times from *nix + # timestamps to ISO 8601 formatted dates by ignoring old cached + # values. + return + + expires = timeutils.normalize_time(expires) + utcnow = timeutils.utcnow() + if ignore_expires or utcnow < expires: self.LOG.debug('Returning cached token %s', token_id) return data else: From 7beb36518e83387038787658414b168434ff67bf Mon Sep 17 00:00:00 2001 From: ZhiQiang Fan Date: Fri, 20 Sep 2013 04:30:41 +0800 Subject: [PATCH 060/120] Replace OpenStack LLC with OpenStack Foundation Some files still use trademark OpenStack LLC in header, which should be changed to OpenStack Foundation. NOTE: tools/install_venv.py is not touched, should sync with oslo Change-Id: I01d4f6b64cf1a152c4e190407799ce7d53de845f Fixes-Bug: #1214176 --- auth_token.py | 2 +- memcache_crypt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index ea62db34..1a13d800 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010-2012 OpenStack LLC +# Copyright 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/memcache_crypt.py b/memcache_crypt.py index f97d08fc..878f2e94 100644 --- a/memcache_crypt.py +++ b/memcache_crypt.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010-2013 OpenStack LLC +# Copyright 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 69541d015322c9856cb0385de3bc33408a19fd88 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Tue, 15 Oct 2013 11:21:56 +0800 Subject: [PATCH 061/120] Migrate the keystone.common.cms to keystoneclient - Add checking the openssl return code 2, related to following review https://review.openstack.org/#/c/22716/ - Add support set subprocess to the cms, when we already know which subprocess to use. Closes-Bug: #1142574 Change-Id: I3f86e6ca8bb7738f57051ce7f0f5662b20e7a22b --- auth_token.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/auth_token.py b/auth_token.py index ea62db34..78f29bb2 100644 --- a/auth_token.py +++ b/auth_token.py @@ -157,6 +157,7 @@ import netaddr import six from keystoneclient.common import cms +from keystoneclient import exceptions from keystoneclient.middleware import memcache_crypt from keystoneclient.openstack.common import jsonutils from keystoneclient.openstack.common import memorycache @@ -1147,7 +1148,7 @@ class AuthProtocol(object): try: output = cms.cms_verify(data, self.signing_cert_file_name, self.signing_ca_file_name) - except cms.subprocess.CalledProcessError as err: + except exceptions.CertificateConfigError as err: if self.cert_file_missing(err.output, self.signing_cert_file_name): self.fetch_signing_cert() @@ -1156,8 +1157,10 @@ class AuthProtocol(object): self.signing_ca_file_name): self.fetch_ca_cert() continue + raise + except cms.subprocess.CalledProcessError as err: self.LOG.warning('Verify error: %s' % err) - raise err + raise return output def verify_signed_token(self, signed_text): @@ -1255,8 +1258,10 @@ class AuthProtocol(object): with open(self.signing_cert_file_name, 'w') as certfile: certfile.write(data) + if response.status_code != 200: + raise exceptions.CertificateConfigError(response.text) + try: - #todo check response try: write_cert_file(response.text) except IOError: @@ -1271,8 +1276,10 @@ class AuthProtocol(object): path = self.auth_admin_prefix.rstrip('/') + '/v2.0/certificates/ca' response = self._http_request('GET', path) + if response.status_code != 200: + raise exceptions.CertificateConfigError(response.text) + try: - #todo check response with open(self.signing_ca_file_name, 'w') as certfile: certfile.write(response.text) except (AssertionError, KeyError): From a947c715efbab53a22333773ee2e79098e391f92 Mon Sep 17 00:00:00 2001 From: Kieran Spear Date: Tue, 23 Jul 2013 17:28:09 +1000 Subject: [PATCH 062/120] Convert revocation list file last modified to UTC On a restart of a service using auth_token middleware, the last modified time of the revocation list file is checked to decide whether to get the fresh list from keystone. In server timezones that are ahead of UTC, this compares a local time with UTC. This means whenever a service is restarted it doesn't update the revocation list for the length of the timezone offset from UTC. This change converts the last modified time to UTC when it's first read, so the comparison is valid. Closes-bug: 1204000 Change-Id: I623b6273beb56f8da2a8649a10a64318da8cd6bc --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 1a13d800..3d3cb986 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1191,7 +1191,7 @@ class AuthProtocol(object): # modification time. if os.path.exists(self.revoked_file_name): mtime = os.path.getmtime(self.revoked_file_name) - fetched_time = datetime.datetime.fromtimestamp(mtime) + fetched_time = datetime.datetime.utcfromtimestamp(mtime) # Otherwise the list will need to be fetched. else: fetched_time = datetime.datetime.min From 44cc2e4a765e84b1cb1faa68cf19d340b3615df4 Mon Sep 17 00:00:00 2001 From: guang-yee Date: Fri, 11 Oct 2013 14:08:57 -0700 Subject: [PATCH 063/120] Opt-out of service catalog Introducing a config option 'include_service_catalog' to indicate whether service catalog is needed. If the 'include_service_catalog' option is set to False, middleware will not ask for service catalog on token validation and will not set the X-Service-Catalog header. This option is backward compatible as it is default to True. DocImpact Fixed bug 1228317 Change-Id: Id8c410a7ae0443ac425d20cb9c6a24ee5bb2cb8d --- auth_token.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/auth_token.py b/auth_token.py index 1a13d800..332f8ac5 100644 --- a/auth_token.py +++ b/auth_token.py @@ -291,7 +291,13 @@ opts = [ default=None, secret=True, help='(optional, mandatory if memcache_security_strategy is' - ' defined) this string is used for key derivation.') + ' defined) this string is used for key derivation.'), + cfg.BoolOpt('include_service_catalog', + default=True, + help='(optional) indicate whether to set the X-Service-Catalog' + ' header. If False, middleware will not ask for service' + ' catalog on token validation and will not set the' + ' X-Service-Catalog header.') ] CONF.register_opts(opts, group='keystone_authtoken') @@ -461,6 +467,9 @@ class AuthProtocol(object): self.http_request_max_retries = \ self._conf_get('http_request_max_retries') + self.include_service_catalog = self._conf_get( + 'include_service_catalog') + def _assert_valid_memcache_protection_config(self): if self._memcache_security_strategy: if self._memcache_security_strategy not in ('MAC', 'ENCRYPT'): @@ -921,11 +930,9 @@ class AuthProtocol(object): self.LOG.debug("Received request from user: %s with project_id : %s" " and roles: %s ", user_id, project_id, roles) - try: + if self.include_service_catalog and catalog_key in catalog_root: catalog = catalog_root[catalog_key] rval['X-Service-Catalog'] = jsonutils.dumps(catalog) - except KeyError: - pass return rval @@ -1090,9 +1097,13 @@ class AuthProtocol(object): if self.auth_version == 'v3.0': headers = {'X-Auth-Token': self.get_admin_token(), 'X-Subject-Token': safe_quote(user_token)} + path = '/v3/auth/tokens' + if not self.include_service_catalog: + # NOTE(gyee): only v3 API support this option + path = path + '?nocatalog' response, data = self._json_request( 'GET', - '/v3/auth/tokens', + path, additional_headers=headers) else: headers = {'X-Auth-Token': self.get_admin_token()} From a275d9d56a9925a57db8727d64c1a7341f704a5e Mon Sep 17 00:00:00 2001 From: huangtianhua Date: Mon, 25 Nov 2013 18:30:28 +0800 Subject: [PATCH 064/120] Fix typo in keystoneclient hypen-separated --> hyphen-separated initialzied --> initialized did't --> didn't sematics --> semantics Change-Id: I79841b76fdf7a267e325b8b9d917900ccb393c02 Closes-Bug: #1254660 --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 332f8ac5..327f9d67 100644 --- a/auth_token.py +++ b/auth_token.py @@ -444,7 +444,7 @@ class AuthProtocol(object): # Token caching via memcache self._cache = None - self._cache_initialized = False # cache already initialzied? + self._cache_initialized = False # cache already initialized? # memcache value treatment, ENCRYPT or MAC self._memcache_security_strategy = \ self._conf_get('memcache_security_strategy') From b1d043789965e8b2685cd9c92fceab5f033c675b Mon Sep 17 00:00:00 2001 From: Peter Portante Date: Wed, 20 Nov 2013 17:10:59 -0500 Subject: [PATCH 065/120] Do not format messages before they are logged The python logging facility allows for the logging message to contain string formatting commands, and will format that string with any additional arguments provided. It will only perform this formatting if the logging level allows. So if you have: A. logging.debug("this %s that %s", this, that) B. logging.debug("this %s that %s" % (this, that)) Then A only formats the message if the debug logging level is set, where as B will always format the message. Since this filter is often on the fast-path for swift requests, it will help the small object case to avoid any extra possible work. Change-Id: I51414dc6577df50d5573a0f917e0656c4ae99520 --- auth_token.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/auth_token.py b/auth_token.py index 2a3de1e0..984b1114 100644 --- a/auth_token.py +++ b/auth_token.py @@ -424,7 +424,7 @@ class AuthProtocol(object): self.signing_dirname = self._conf_get('signing_dir') if self.signing_dirname is None: self.signing_dirname = tempfile.mkdtemp(prefix='keystone-signing-') - self.LOG.info('Using %s as cache directory for signing certificate' % + self.LOG.info('Using %s as cache directory for signing certificate', self.signing_dirname) self.verify_signing_dir() @@ -541,7 +541,7 @@ class AuthProtocol(object): self.LOG.warning("Old keystone installation found...assuming v2.0") versions.append("v2.0") elif response.status_code != 300: - self.LOG.error('Unable to get version info from keystone: %s' % + self.LOG.error('Unable to get version info from keystone: %s', response.status_code) raise ServiceError('Unable to get version info from keystone') else: @@ -591,7 +591,7 @@ class AuthProtocol(object): return self._reject_request(env, start_response) except ServiceError as e: - self.LOG.critical('Unable to obtain admin token: %s' % e) + self.LOG.critical('Unable to obtain admin token: %s', e) resp = MiniResp('Service unavailable', env) start_response('503 Service Unavailable', resp.headers) return resp.body @@ -623,7 +623,7 @@ class AuthProtocol(object): 'X-Tenant', 'X-Role', ) - self.LOG.debug('Removing headers from request environment: %s' % + self.LOG.debug('Removing headers from request environment: %s', ','.join(auth_headers)) self._remove_headers(env, auth_headers) @@ -713,7 +713,7 @@ class AuthProtocol(object): self.LOG.error('HTTP connection exception: %s', e) raise NetworkError('Unable to communicate with keystone') # NOTE(vish): sleep 0.5, 1, 2 - self.LOG.warn('Retrying on HTTP connection exception: %s' % e) + self.LOG.warn('Retrying on HTTP connection exception: %s', e) time.sleep(2.0 ** retry / 2) retry += 1 @@ -1123,7 +1123,7 @@ class AuthProtocol(object): 'Keystone rejected admin token %s, resetting', headers) self.admin_token = None else: - self.LOG.error('Bad response code while validating token: %s' % + self.LOG.error('Bad response code while validating token: %s', response.status_code) if retry: self.LOG.info('Retrying validation') @@ -1170,7 +1170,7 @@ class AuthProtocol(object): continue raise except cms.subprocess.CalledProcessError as err: - self.LOG.warning('Verify error: %s' % err) + self.LOG.warning('Verify error: %s', err) raise return output @@ -1187,14 +1187,15 @@ class AuthProtocol(object): if not os.access(self.signing_dirname, os.W_OK): raise ConfigurationError( 'unable to access signing_dir %s' % self.signing_dirname) - if os.stat(self.signing_dirname).st_uid != os.getuid(): + uid = os.getuid() + if os.stat(self.signing_dirname).st_uid != uid: self.LOG.warning( - 'signing_dir is not owned by %s' % os.getuid()) + 'signing_dir is not owned by %s', uid) current_mode = stat.S_IMODE(os.stat(self.signing_dirname).st_mode) if current_mode != stat.S_IRWXU: self.LOG.warning( - 'signing_dir mode is %s instead of %s' % - (oct(current_mode), oct(stat.S_IRWXU))) + 'signing_dir mode is %s instead of %s', + oct(current_mode), oct(stat.S_IRWXU)) else: os.makedirs(self.signing_dirname, stat.S_IRWXU) From 8d66c48a8eb53bd2bb1a2bb0b18a0472bdf8a88a Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Thu, 12 Dec 2013 14:25:06 +0100 Subject: [PATCH 066/120] Python3: replace urllib by six.moves.urllib This makes the code compatible with both Python 2 and 3. Change-Id: I721a5567842f2df6ce2a8af501787204daba3082 --- auth_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index ada4aa3c..984a1190 100644 --- a/auth_token.py +++ b/auth_token.py @@ -151,10 +151,10 @@ import requests import stat import tempfile import time -import urllib import netaddr import six +from six.moves import urllib from keystoneclient.common import cms from keystoneclient import exceptions @@ -343,7 +343,7 @@ def confirm_token_not_expired(data): def safe_quote(s): """URL-encode strings that are not already URL-encoded.""" - return urllib.quote(s) if s == urllib.unquote(s) else s + return urllib.parse.quote(s) if s == urllib.parse.unquote(s) else s class InvalidUserToken(Exception): From d5f0fed9d2846c106fec794df1b79fa993a028d2 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Tue, 17 Dec 2013 23:05:12 +0100 Subject: [PATCH 067/120] Rename instead of writing directly to revoked file Make the operation more atomic with multiple writers. Closes-Bug: 1261554 Change-Id: I990a2ba28d9a2a1d01300dcd33266956d059afa3 --- auth_token.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 984a1190..36c8fba6 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1241,8 +1241,11 @@ class AuthProtocol(object): """ self._token_revocation_list = jsonutils.loads(value) self.token_revocation_list_fetched_time = timeutils.utcnow() - with open(self.revoked_file_name, 'w') as f: + + with tempfile.NamedTemporaryFile(dir=self.signing_dirname, + delete=False) as f: f.write(value) + os.rename(f.name, self.revoked_file_name) def fetch_revocation_list(self, retry=True): headers = {'X-Auth-Token': self.get_admin_token()} From 4e18269b8078ce6702b996213336f5da37bfd181 Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Thu, 3 Oct 2013 17:24:19 +1000 Subject: [PATCH 068/120] Verify token binding in auth_token middleware The server side has had token bind checking implemented for a while. Now that it is possible to generate bound tokens auth_token middleware should be capable of verifying them. Change-Id: I4f9c5855ab3102333b0738864c506e2501bf9c7e --- auth_token.py | 98 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 4 deletions(-) diff --git a/auth_token.py b/auth_token.py index 36c8fba6..ade945b8 100644 --- a/auth_token.py +++ b/auth_token.py @@ -298,7 +298,17 @@ opts = [ help='(optional) indicate whether to set the X-Service-Catalog' ' header. If False, middleware will not ask for service' ' catalog on token validation and will not set the' - ' X-Service-Catalog header.') + ' X-Service-Catalog header.'), + cfg.StrOpt('enforce_token_bind', + default='permissive', + help='Used to control the use and type of token binding. Can' + ' be set to: "disabled" to not check token binding.' + ' "permissive" (default) to validate binding information if the' + ' bind type is of a form known to the server and ignore it if' + ' not. "strict" like "permissive" but if the bind type is' + ' unknown the token will be rejected. "required" any form of' + ' token binding is needed to be allowed. Finally the name of a' + ' binding method that must be present in tokens.'), ] CONF.register_opts(opts, group='keystone_authtoken') @@ -306,6 +316,14 @@ LIST_OF_VERSIONS_TO_ATTEMPT = ['v2.0', 'v3.0'] CACHE_KEY_TEMPLATE = 'tokens/%s' +class BIND_MODE: + DISABLED = 'disabled' + PERMISSIVE = 'permissive' + STRICT = 'strict' + REQUIRED = 'required' + KERBEROS = 'kerberos' + + def will_expire_soon(expiry): """Determines if expiration is about to occur. @@ -574,7 +592,7 @@ class AuthProtocol(object): try: self._remove_auth_headers(env) user_token = self._get_user_token_from_header(env) - token_info = self._validate_user_token(user_token) + token_info = self._validate_user_token(user_token, env) env['keystone.token_info'] = token_info user_headers = self._build_user_headers(token_info) self._add_headers(env, user_headers) @@ -797,7 +815,7 @@ class AuthProtocol(object): "Unable to parse expiration time from token: %s", data) raise ServiceError('invalid json response') - def _validate_user_token(self, user_token, retry=True): + def _validate_user_token(self, user_token, env, retry=True): """Authenticate user using PKI :param user_token: user's token id @@ -820,6 +838,7 @@ class AuthProtocol(object): else: data = self.verify_uuid_token(user_token, retry) expires = confirm_token_not_expired(data) + self._confirm_token_bind(data, env) self._cache_put(token_id, data, expires) return data except NetworkError: @@ -1058,6 +1077,77 @@ class AuthProtocol(object): data_to_store, timeout=self.token_cache_time) + def _invalid_user_token(self, msg=False): + # NOTE(jamielennox): use False as the default so that None is valid + if msg is False: + msg = 'Token authorization failed' + + raise InvalidUserToken(msg) + + def _confirm_token_bind(self, data, env): + bind_mode = self._conf_get('enforce_token_bind') + + if bind_mode == BIND_MODE.DISABLED: + return + + try: + if _token_is_v2(data): + bind = data['access']['token']['bind'] + elif _token_is_v3(data): + bind = data['token']['bind'] + else: + self._invalid_user_token() + except KeyError: + bind = {} + + # permissive and strict modes don't require there to be a bind + permissive = bind_mode in (BIND_MODE.PERMISSIVE, BIND_MODE.STRICT) + + if not bind: + if permissive: + # no bind provided and none required + return + else: + self.LOG.info("No bind information present in token.") + self._invalid_user_token() + + # get the named mode if bind_mode is not one of the predefined + if permissive or bind_mode == BIND_MODE.REQUIRED: + name = None + else: + name = bind_mode + + if name and name not in bind: + self.LOG.info("Named bind mode %s not in bind information", name) + self._invalid_user_token() + + for bind_type, identifier in six.iteritems(bind): + if bind_type == BIND_MODE.KERBEROS: + if not env.get('AUTH_TYPE', '').lower() == 'negotiate': + self.LOG.info("Kerberos credentials required and " + "not present.") + self._invalid_user_token() + + if not env.get('REMOTE_USER') == identifier: + self.LOG.info("Kerberos credentials do not match " + "those in bind.") + self._invalid_user_token() + + self.LOG.debug("Kerberos bind authentication successful.") + + elif bind_mode == BIND_MODE.PERMISSIVE: + self.LOG.debug("Ignoring Unknown bind for permissive mode: " + "%(bind_type)s: %(identifier)s.", + {'bind_type': bind_type, + 'identifier': identifier}) + + else: + self.LOG.info("Couldn't verify unknown bind: %(bind_type)s: " + "%(identifier)s.", + {'bind_type': bind_type, + 'identifier': identifier}) + self._invalid_user_token() + def _cache_put(self, token_id, data, expires): """Put token data into the cache. @@ -1127,7 +1217,7 @@ class AuthProtocol(object): response.status_code) if retry: self.LOG.info('Retrying validation') - return self._validate_user_token(user_token, False) + return self._validate_user_token(user_token, env, False) else: self.LOG.warn("Invalid user token: %s. Keystone response: %s.", user_token, data) From c2ed157b51aa82906741378df9372a18dc3b3b85 Mon Sep 17 00:00:00 2001 From: Eric Guo Date: Fri, 17 Jan 2014 20:13:24 +0800 Subject: [PATCH 069/120] Adjust import items according to hacking import rule This patch adjust import items and add missing blank lines acording to http://docs.openstack.org/developer/hacking/#imports {{stdlib imports in human alphabetical order}} \n {{third-party lib imports in human alphabetical order}} \n {{project imports in human alphabetical order}} \n \n {{begin your code}} hacking project also enforce some checks for import group. Let make the change in keytoneclient Change-Id: Ic83bd5ee426905588f4a2d555851a9a01fc69f02 --- auth_token.py | 1 + 1 file changed, 1 insertion(+) diff --git a/auth_token.py b/auth_token.py index ade945b8..53a136de 100644 --- a/auth_token.py +++ b/auth_token.py @@ -164,6 +164,7 @@ from keystoneclient.openstack.common import memorycache from keystoneclient.openstack.common import timeutils from keystoneclient import utils + CONF = None # to pass gate before oslo-config is deployed everywhere, # try application copies first From d21e1743668f2c32242b8b04cf933a474427f9fc Mon Sep 17 00:00:00 2001 From: Eric Guo Date: Thu, 23 Jan 2014 16:30:19 +0800 Subject: [PATCH 070/120] Fix typos in documents and comments Fix typos detected by toolkit misspellings. * pip install misspellings * git ls-files | grep -v locale | misspellings -f - Change-Id: Ifbbc29537d9d129aad238de6c37718c4fbb8349b --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index ade945b8..ff1135f2 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1063,7 +1063,7 @@ class AuthProtocol(object): cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) data_to_store = memcache_crypt.protect_data(keys, serialized_data) - # Historically the swift cache conection used the argument + # Historically the swift cache connection used the argument # timeout= for the cache timeout, but this has been unified # with the official python memcache client with time= since # grizzly, we still need to handle folsom for a while until From 44e146c7d28c23dd36c959f0017da48a2ddf6338 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Fri, 18 Oct 2013 15:31:20 -0500 Subject: [PATCH 071/120] Copy s3_token middleware from keystone The s3_token middleware was in keystone but it should be in python-keystoneclient. As a first step to removing s3_token from keystone, copy it to python-keystoneclient. Also copy the tests. A couple of changes were required - Changed oslo-incubator imports from keystone to keystoneclient. - Changed logging from oslo-incubator to use Python logging and configured the same way as the auth_token middleware. I checked and I didn't see anything to be added to requirements.txt or test-requirements.txt. bp s3-token-to-keystoneclient Change-Id: I64fef4101180e5aa661442d538c3237bdad0c37c Closes-Bug: #1178741 --- s3_token.py | 263 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 s3_token.py diff --git a/s3_token.py b/s3_token.py new file mode 100644 index 00000000..f35b1545 --- /dev/null +++ b/s3_token.py @@ -0,0 +1,263 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack Foundation +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011,2012 Akira YOSHIYAMA +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This source code is based ./auth_token.py and ./ec2_token.py. +# See them for their copyright. + +""" +S3 TOKEN MIDDLEWARE + +This WSGI component: + +* Get a request from the swift3 middleware with an S3 Authorization + access key. +* Validate s3 token in Keystone. +* Transform the account name to AUTH_%(tenant_name). + +""" + +import httplib +import logging +import urllib +import webob + +from keystoneclient.openstack.common import jsonutils + + +PROTOCOL_NAME = 'S3 Token Authentication' + + +# TODO(kun): remove it after oslo merge this. +def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): + """Validate and split the given HTTP request path. + + **Examples**:: + + ['a'] = split_path('/a') + ['a', None] = split_path('/a', 1, 2) + ['a', 'c'] = split_path('/a/c', 1, 2) + ['a', 'c', 'o/r'] = split_path('/a/c/o/r', 1, 3, True) + + :param path: HTTP Request path to be split + :param minsegs: Minimum number of segments to be extracted + :param maxsegs: Maximum number of segments to be extracted + :param rest_with_last: If True, trailing data will be returned as part + of last segment. If False, and there is + trailing data, raises ValueError. + :returns: list of segments with a length of maxsegs (non-existant + segments will return as None) + :raises: ValueError if given an invalid path + """ + if not maxsegs: + maxsegs = minsegs + if minsegs > maxsegs: + raise ValueError('minsegs > maxsegs: %d > %d' % (minsegs, maxsegs)) + if rest_with_last: + segs = path.split('/', maxsegs) + minsegs += 1 + maxsegs += 1 + count = len(segs) + if (segs[0] or count < minsegs or count > maxsegs or + '' in segs[1:minsegs]): + raise ValueError('Invalid path: %s' % urllib.quote(path)) + else: + minsegs += 1 + maxsegs += 1 + segs = path.split('/', maxsegs) + count = len(segs) + if (segs[0] or count < minsegs or count > maxsegs + 1 or + '' in segs[1:minsegs] or + (count == maxsegs + 1 and segs[maxsegs])): + raise ValueError('Invalid path: %s' % urllib.quote(path)) + segs = segs[1:maxsegs] + segs.extend([None] * (maxsegs - 1 - len(segs))) + return segs + + +class ServiceError(Exception): + pass + + +class S3Token(object): + """Auth Middleware that handles S3 authenticating client calls.""" + + def __init__(self, app, conf): + """Common initialization code.""" + self.app = app + self.logger = logging.getLogger(conf.get('log_name', __name__)) + self.logger.debug('Starting the %s component' % PROTOCOL_NAME) + self.reseller_prefix = conf.get('reseller_prefix', 'AUTH_') + # where to find the auth service (we use this to validate tokens) + self.auth_host = conf.get('auth_host') + self.auth_port = int(conf.get('auth_port', 35357)) + self.auth_protocol = conf.get('auth_protocol', 'https') + if self.auth_protocol == 'http': + self.http_client_class = httplib.HTTPConnection + else: + self.http_client_class = httplib.HTTPSConnection + # SSL + self.cert_file = conf.get('certfile') + self.key_file = conf.get('keyfile') + + def deny_request(self, code): + error_table = { + 'AccessDenied': (401, 'Access denied'), + 'InvalidURI': (400, 'Could not parse the specified URI'), + } + resp = webob.Response(content_type='text/xml') + resp.status = error_table[code][0] + resp.body = error_table[code][1] + resp.body = ('\r\n' + '\r\n %s\r\n ' + '%s\r\n\r\n' % + (code, error_table[code][1])) + return resp + + def _json_request(self, creds_json): + headers = {'Content-Type': 'application/json'} + if self.auth_protocol == 'http': + conn = self.http_client_class(self.auth_host, self.auth_port) + else: + conn = self.http_client_class(self.auth_host, + self.auth_port, + self.key_file, + self.cert_file) + try: + conn.request('POST', '/v2.0/s3tokens', + body=creds_json, + headers=headers) + response = conn.getresponse() + output = response.read() + except Exception as e: + self.logger.info('HTTP connection exception: %s' % e) + resp = self.deny_request('InvalidURI') + raise ServiceError(resp) + finally: + conn.close() + + if response.status < 200 or response.status >= 300: + self.logger.debug('Keystone reply error: status=%s reason=%s' % + (response.status, response.reason)) + resp = self.deny_request('AccessDenied') + raise ServiceError(resp) + + return (response, output) + + def __call__(self, environ, start_response): + """Handle incoming request. authenticate and send downstream.""" + req = webob.Request(environ) + self.logger.debug('Calling S3Token middleware.') + + try: + parts = split_path(req.path, 1, 4, True) + version, account, container, obj = parts + except ValueError: + msg = 'Not a path query, skipping.' + self.logger.debug(msg) + return self.app(environ, start_response) + + # Read request signature and access id. + if 'Authorization' not in req.headers: + msg = 'No Authorization header. skipping.' + self.logger.debug(msg) + return self.app(environ, start_response) + + token = req.headers.get('X-Auth-Token', + req.headers.get('X-Storage-Token')) + if not token: + msg = 'You did not specify a auth or a storage token. skipping.' + self.logger.debug(msg) + return self.app(environ, start_response) + + auth_header = req.headers['Authorization'] + try: + access, signature = auth_header.split(' ')[-1].rsplit(':', 1) + except ValueError: + msg = 'You have an invalid Authorization header: %s' + self.logger.debug(msg % (auth_header)) + return self.deny_request('InvalidURI')(environ, start_response) + + # NOTE(chmou): This is to handle the special case with nova + # when we have the option s3_affix_tenant. We will force it to + # connect to another account than the one + # authenticated. Before people start getting worried about + # security, I should point that we are connecting with + # username/token specified by the user but instead of + # connecting to its own account we will force it to go to an + # another account. In a normal scenario if that user don't + # have the reseller right it will just fail but since the + # reseller account can connect to every account it is allowed + # by the swift_auth middleware. + force_tenant = None + if ':' in access: + access, force_tenant = access.split(':') + + # Authenticate request. + creds = {'credentials': {'access': access, + 'token': token, + 'signature': signature}} + creds_json = jsonutils.dumps(creds) + self.logger.debug('Connecting to Keystone sending this JSON: %s' % + creds_json) + # NOTE(vish): We could save a call to keystone by having + # keystone return token, tenant, user, and roles + # from this call. + # + # NOTE(chmou): We still have the same problem we would need to + # change token_auth to detect if we already + # identified and not doing a second query and just + # pass it through to swiftauth in this case. + try: + resp, output = self._json_request(creds_json) + except ServiceError as e: + resp = e.args[0] + msg = 'Received error, exiting middleware with error: %s' + self.logger.debug(msg % (resp.status)) + return resp(environ, start_response) + + self.logger.debug('Keystone Reply: Status: %d, Output: %s' % ( + resp.status, output)) + + try: + identity_info = jsonutils.loads(output) + token_id = str(identity_info['access']['token']['id']) + tenant = identity_info['access']['token']['tenant'] + except (ValueError, KeyError): + error = 'Error on keystone reply: %d %s' + self.logger.debug(error % (resp.status, str(output))) + return self.deny_request('InvalidURI')(environ, start_response) + + req.headers['X-Auth-Token'] = token_id + tenant_to_connect = force_tenant or tenant['id'] + self.logger.debug('Connecting with tenant: %s' % (tenant_to_connect)) + new_tenant_name = '%s%s' % (self.reseller_prefix, tenant_to_connect) + environ['PATH_INFO'] = environ['PATH_INFO'].replace(account, + new_tenant_name) + return self.app(environ, start_response) + + +def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def auth_filter(app): + return S3Token(app, conf) + return auth_filter From e3ef05b4664c60ba84330569c846fde1c25a8574 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Fri, 17 Jan 2014 01:12:26 +0100 Subject: [PATCH 072/120] Python 3: Fix an str vs bytes issue in tempfile In Python 3, one cannot write: from tempfile import NamedTemporaryFile with NamedTemporaryFile() as f: f.write('foobar') The input of f.write() must be bytes. Encode it when necessary in middleware/auth_token.py, in a way that is compatible with Python 2. Change-Id: Ib60afbc5e01c35f59cd7c9b68bfedb10ad897ff9 --- auth_token.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index ff1135f2..e442442b 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1316,7 +1316,8 @@ class AuthProtocol(object): if list_is_current: # Load the list from disk if required if not self._token_revocation_list: - with open(self.revoked_file_name, 'r') as f: + open_kwargs = {'encoding': 'utf-8'} if six.PY3 else {} + with open(self.revoked_file_name, 'r', **open_kwargs) as f: self._token_revocation_list = jsonutils.loads(f.read()) else: self.token_revocation_list = self.fetch_revocation_list() @@ -1334,6 +1335,10 @@ class AuthProtocol(object): with tempfile.NamedTemporaryFile(dir=self.signing_dirname, delete=False) as f: + # In Python2, encoding is slow so the following check avoids it if + # it is not absolutely necessary. + if isinstance(value, six.text_type): + value = value.encode('utf-8') f.write(value) os.rename(f.name, self.revoked_file_name) From 8cafe302adea3560f08d47d5e43e918d92f92b06 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Sun, 2 Feb 2014 05:28:15 +0100 Subject: [PATCH 073/120] Remove support for old Swift memcache interface. Since this has been removed since a year, it's time to clean this up. Change-Id: I6681a31f675f2178285dadea876af1fdb7a8a4c3 --- auth_token.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/auth_token.py b/auth_token.py index e442442b..b4183f0d 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1063,19 +1063,9 @@ class AuthProtocol(object): cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) data_to_store = memcache_crypt.protect_data(keys, serialized_data) - # Historically the swift cache connection used the argument - # timeout= for the cache timeout, but this has been unified - # with the official python memcache client with time= since - # grizzly, we still need to handle folsom for a while until - # this could get removed. - try: - self._cache.set(cache_key, - data_to_store, - time=self.token_cache_time) - except(TypeError): - self._cache.set(cache_key, - data_to_store, - timeout=self.token_cache_time) + self._cache.set(cache_key, + data_to_store, + time=self.token_cache_time) def _invalid_user_token(self, msg=False): # NOTE(jamielennox): use False as the default so that None is valid From a84e53e42f3e2d83815d9c6d54efc3084d8f4f63 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Sun, 2 Feb 2014 11:21:15 -0600 Subject: [PATCH 074/120] Update reference to middlewarearchitecture doc auth_token was referring to the middlewarearchitecture doc in keystone developer docs rather than python-keystoneclient. Change-Id: I7bfb3595097cc4a2a1061efe5c4480e06a499edb --- auth_token.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 8a6add57..9fe3b475 100644 --- a/auth_token.py +++ b/auth_token.py @@ -28,7 +28,8 @@ This WSGI component: * Collects and forwards identity information based on a valid token such as user name, tenant, etc -Refer to: http://keystone.openstack.org/middlewarearchitecture.html +Refer to: http://docs.openstack.org/developer/python-keystoneclient/ +middlewarearchitecture.html HEADERS ------- From 38107c262351fd713ed87284338cbbc2cf5d8a5c Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Thu, 16 Jan 2014 20:33:52 +0100 Subject: [PATCH 075/120] Python 3: make tests from v2_0/test_access.py pass This fixes calls to the hash_signed_token() and cms_hash_token() functions, by making sure they are given bytes. Change-Id: I83ac48a845cd09150b01afad6f0549ee83c20ddd --- auth_token.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/auth_token.py b/auth_token.py index 8a6add57..6a8f6c19 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1232,6 +1232,8 @@ class AuthProtocol(object): if not revoked_tokens: return revoked_ids = (x['id'] for x in revoked_tokens) + if isinstance(signed_text, six.text_type): + signed_text = signed_text.encode('utf-8') token_id = utils.hash_signed_token(signed_text) for revoked_id in revoked_ids: if token_id == revoked_id: From 324570b4012b3d4109d22aa2083be351869a80e7 Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Mon, 3 Feb 2014 13:14:53 +1000 Subject: [PATCH 076/120] Use requests library in S3 middleware Convert the S3 token middleware to use the requests library. It does not validate certificates as part of this patch to keep new functionality isolated. Change-Id: I0f26c80a969b919de80410af0a80920bd8493191 Closes-Bug: #1275598 --- s3_token.py | 66 ++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/s3_token.py b/s3_token.py index f35b1545..59dd331b 100644 --- a/s3_token.py +++ b/s3_token.py @@ -33,11 +33,12 @@ This WSGI component: """ -import httplib import logging import urllib import webob +import requests + from keystoneclient.openstack.common import jsonutils @@ -105,16 +106,26 @@ class S3Token(object): self.logger.debug('Starting the %s component' % PROTOCOL_NAME) self.reseller_prefix = conf.get('reseller_prefix', 'AUTH_') # where to find the auth service (we use this to validate tokens) - self.auth_host = conf.get('auth_host') - self.auth_port = int(conf.get('auth_port', 35357)) - self.auth_protocol = conf.get('auth_protocol', 'https') - if self.auth_protocol == 'http': - self.http_client_class = httplib.HTTPConnection - else: - self.http_client_class = httplib.HTTPSConnection + + auth_host = conf.get('auth_host') + auth_port = int(conf.get('auth_port', 35357)) + auth_protocol = conf.get('auth_protocol', 'https') + + self.request_uri = '%s://%s:%s' % (auth_protocol, auth_host, auth_port) + # SSL - self.cert_file = conf.get('certfile') - self.key_file = conf.get('keyfile') + insecure = conf.get('insecure', False) + cert_file = conf.get('certfile') + key_file = conf.get('keyfile') + + if insecure: + self.verify = False + elif cert_file and key_file: + self.verify = (cert_file, key_file) + elif cert_file: + self.verify = cert_file + else: + self.verify = None def deny_request(self, code): error_table = { @@ -132,33 +143,22 @@ class S3Token(object): def _json_request(self, creds_json): headers = {'Content-Type': 'application/json'} - if self.auth_protocol == 'http': - conn = self.http_client_class(self.auth_host, self.auth_port) - else: - conn = self.http_client_class(self.auth_host, - self.auth_port, - self.key_file, - self.cert_file) try: - conn.request('POST', '/v2.0/s3tokens', - body=creds_json, - headers=headers) - response = conn.getresponse() - output = response.read() - except Exception as e: + response = requests.post('%s/v2.0/s3tokens' % self.request_uri, + headers=headers, data=creds_json, + verify=self.verify) + except requests.exceptions.RequestException as e: self.logger.info('HTTP connection exception: %s' % e) resp = self.deny_request('InvalidURI') raise ServiceError(resp) - finally: - conn.close() - if response.status < 200 or response.status >= 300: + if response.status_code < 200 or response.status_code >= 300: self.logger.debug('Keystone reply error: status=%s reason=%s' % - (response.status, response.reason)) + (response.status_code, response.reason)) resp = self.deny_request('AccessDenied') raise ServiceError(resp) - return (response, output) + return response def __call__(self, environ, start_response): """Handle incoming request. authenticate and send downstream.""" @@ -225,23 +225,23 @@ class S3Token(object): # identified and not doing a second query and just # pass it through to swiftauth in this case. try: - resp, output = self._json_request(creds_json) + resp = self._json_request(creds_json) except ServiceError as e: resp = e.args[0] msg = 'Received error, exiting middleware with error: %s' - self.logger.debug(msg % (resp.status)) + self.logger.debug(msg % (resp.status_code)) return resp(environ, start_response) self.logger.debug('Keystone Reply: Status: %d, Output: %s' % ( - resp.status, output)) + resp.status_code, resp.content)) try: - identity_info = jsonutils.loads(output) + identity_info = resp.json() token_id = str(identity_info['access']['token']['id']) tenant = identity_info['access']['token']['tenant'] except (ValueError, KeyError): error = 'Error on keystone reply: %d %s' - self.logger.debug(error % (resp.status, str(output))) + self.logger.debug(error % (resp.status_code, str(resp.content))) return self.deny_request('InvalidURI')(environ, start_response) req.headers['X-Auth-Token'] = token_id From c7b30342b4abbe212f9eecac71809ffecfa7b809 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Thu, 23 Jan 2014 23:02:38 +0100 Subject: [PATCH 077/120] Python 3: call functions from memcache_crypt.py with bytes as input These functions expect bytes as input, but in Python 3 they were given text strings. Change-Id: I39fa15b8d5d56dc536e0bd71a50cf27da3d03046 --- auth_token.py | 14 +++++++++++--- memcache_crypt.py | 40 ++++++++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/auth_token.py b/auth_token.py index 8a6add57..88dca186 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1022,6 +1022,8 @@ class AuthProtocol(object): # Note that 'invalid' and (data, expires) are the only # valid types of serialized cache entries, so there is not # a collision with jsonutils.loads(serialized) == None. + if not isinstance(serialized, six.string_types): + serialized = serialized.decode('utf-8') cached = jsonutils.loads(serialized) if cached == 'invalid': self.LOG.debug('Cached Token %s is marked unauthorized', @@ -1053,14 +1055,20 @@ class AuthProtocol(object): """ serialized_data = jsonutils.dumps(data) + if isinstance(serialized_data, six.text_type): + serialized_data = serialized_data.encode('utf-8') if self._memcache_security_strategy is None: cache_key = CACHE_KEY_TEMPLATE % token_id data_to_store = serialized_data else: + secret_key = self._memcache_secret_key + if isinstance(secret_key, six.string_types): + secret_key = secret_key.encode('utf-8') + security_strategy = self._memcache_security_strategy + if isinstance(security_strategy, six.string_types): + security_strategy = security_strategy.encode('utf-8') keys = memcache_crypt.derive_keys( - token_id, - self._memcache_secret_key, - self._memcache_security_strategy) + token_id, secret_key, security_strategy) cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) data_to_store = memcache_crypt.protect_data(keys, serialized_data) diff --git a/memcache_crypt.py b/memcache_crypt.py index 878f2e94..8bae5068 100644 --- a/memcache_crypt.py +++ b/memcache_crypt.py @@ -35,6 +35,8 @@ import hashlib import hmac import math import os +import six +import sys # make sure pycrypto is available try: @@ -82,19 +84,26 @@ def assert_crypto_availability(f): return wrapper -def constant_time_compare(first, second): - """Returns True if both string inputs are equal, otherwise False. +if sys.version_info >= (3, 3): + constant_time_compare = hmac.compare_digest +else: + def constant_time_compare(first, second): + """Returns True if both string inputs are equal, otherwise False. - This function should take a constant amount of time regardless of - how many characters in the strings match. + This function should take a constant amount of time regardless of + how many characters in the strings match. - """ - if len(first) != len(second): - return False - result = 0 - for x, y in zip(first, second): - result |= ord(x) ^ ord(y) - return result == 0 + """ + if len(first) != len(second): + return False + result = 0 + if six.PY3 and isinstance(first, bytes) and isinstance(second, bytes): + for x, y in zip(first, second): + result |= x ^ y + else: + for x, y in zip(first, second): + result |= ord(x) ^ ord(y) + return result == 0 def derive_keys(token, secret, strategy): @@ -132,7 +141,7 @@ def encrypt_data(key, data): iv = os.urandom(16) cipher = AES.new(key, AES.MODE_CBC, iv) padding = 16 - len(data) % 16 - return iv + cipher.encrypt(data + chr(padding) * padding) + return iv + cipher.encrypt(data + six.int2byte(padding) * padding) @assert_crypto_availability @@ -147,8 +156,7 @@ def decrypt_data(key, data): # Strip the last n padding bytes where n is the last value in # the plaintext - padding = ord(result[-1]) - return result[:-1 * padding] + return result[:-1 * six.byte2int([result[-1]])] def protect_data(keys, data): @@ -156,7 +164,7 @@ def protect_data(keys, data): protected string suitable for storage in the cache. """ - if keys['strategy'] == 'ENCRYPT': + if keys['strategy'] == b'ENCRYPT': data = encrypt_data(keys['ENCRYPTION'], data) encoded_data = base64.b64encode(data) @@ -188,7 +196,7 @@ def unprotect_data(keys, signed_data): data = base64.b64decode(signed_data[DIGEST_LENGTH_B64:]) # then if necessary decrypt the data - if keys['strategy'] == 'ENCRYPT': + if keys['strategy'] == b'ENCRYPT': data = decrypt_data(keys['ENCRYPTION'], data) return data From f7c56b379d79a0a2e0203b245444d44aaf946d87 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Thu, 6 Feb 2014 17:11:39 +0100 Subject: [PATCH 078/120] Python3: webob.Response.body must be bytes Remove duplicate assignement for 'resp.body'. Use bytes in Python3. Change-Id: Ice1beb997ae401ebb87762e5499970409c63d801 --- s3_token.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/s3_token.py b/s3_token.py index 59dd331b..546fd6aa 100644 --- a/s3_token.py +++ b/s3_token.py @@ -38,6 +38,7 @@ import urllib import webob import requests +import six from keystoneclient.openstack.common import jsonutils @@ -134,11 +135,13 @@ class S3Token(object): } resp = webob.Response(content_type='text/xml') resp.status = error_table[code][0] - resp.body = error_table[code][1] - resp.body = ('\r\n' + error_msg = ('\r\n' '\r\n %s\r\n ' '%s\r\n\r\n' % (code, error_table[code][1])) + if six.PY3: + error_msg = error_msg.encode() + resp.body = error_msg return resp def _json_request(self, creds_json): From 22a1350f8bdf5e4bc5487a22e7a08c6e6927ad6e Mon Sep 17 00:00:00 2001 From: Eric Guo Date: Sat, 8 Feb 2014 22:22:08 +0800 Subject: [PATCH 079/120] Remove vim header We don't need vim modelines in each source file, it can be set in user's vimrc. Change-Id: Ic7a61430a0a320ce6b0c4518d9f5d988e35f8aae Closes-Bug: #1229324 --- auth_token.py | 2 -- memcache_crypt.py | 2 -- s3_token.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/auth_token.py b/auth_token.py index 6a8f6c19..d67f8a74 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010-2012 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/memcache_crypt.py b/memcache_crypt.py index 878f2e94..49e5fd33 100644 --- a/memcache_crypt.py +++ b/memcache_crypt.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2010-2013 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/s3_token.py b/s3_token.py index 59dd331b..c81a6d6d 100644 --- a/s3_token.py +++ b/s3_token.py @@ -1,5 +1,3 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - # Copyright 2012 OpenStack Foundation # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. From 0beea24e05fb376f7a95710a9bb25d47f06af54f Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Mon, 10 Feb 2014 01:41:41 +0100 Subject: [PATCH 080/120] Python3: use six.moves.urllib.parse.quote instead of urllib.quote This makes the code compatible with both python 2 and 3. Change-Id: Ic1c5c8d4a85db138a8402ea4b3208828fc118a40 --- s3_token.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s3_token.py b/s3_token.py index 546fd6aa..72a82e93 100644 --- a/s3_token.py +++ b/s3_token.py @@ -34,11 +34,11 @@ This WSGI component: """ import logging -import urllib import webob import requests import six +from six.moves import urllib from keystoneclient.openstack.common import jsonutils @@ -78,7 +78,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): count = len(segs) if (segs[0] or count < minsegs or count > maxsegs or '' in segs[1:minsegs]): - raise ValueError('Invalid path: %s' % urllib.quote(path)) + raise ValueError('Invalid path: %s' % urllib.parse.quote(path)) else: minsegs += 1 maxsegs += 1 @@ -87,7 +87,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): if (segs[0] or count < minsegs or count > maxsegs + 1 or '' in segs[1:minsegs] or (count == maxsegs + 1 and segs[maxsegs])): - raise ValueError('Invalid path: %s' % urllib.quote(path)) + raise ValueError('Invalid path: %s' % urllib.parse.quote(path)) segs = segs[1:maxsegs] segs.extend([None] * (maxsegs - 1 - len(segs))) return segs From 1eed789fc19bcf3eeaa883b5c78955fc768078a8 Mon Sep 17 00:00:00 2001 From: Cyril Roelandt Date: Tue, 11 Feb 2014 14:58:47 +0100 Subject: [PATCH 081/120] Python: Pass bytes to derive_keys() This call was missed in d71b5b3. Change-Id: I0705dbefa5893e0ff5b786202f58fb254939ae76 --- auth_token.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index e27fb9db..a49459ad 100644 --- a/auth_token.py +++ b/auth_token.py @@ -998,10 +998,16 @@ class AuthProtocol(object): key = CACHE_KEY_TEMPLATE % token_id serialized = self._cache.get(key) else: + secret_key = self._memcache_secret_key + if isinstance(secret_key, six.string_types): + secret_key = secret_key.encode('utf-8') + security_strategy = self._memcache_security_strategy + if isinstance(security_strategy, six.string_types): + security_strategy = security_strategy.encode('utf-8') keys = memcache_crypt.derive_keys( token_id, - self._memcache_secret_key, - self._memcache_security_strategy) + secret_key, + security_strategy) cache_key = CACHE_KEY_TEMPLATE % ( memcache_crypt.get_cache_key(keys)) raw_cached = self._cache.get(cache_key) From ee9247d66fa7ef6ccc3511d2b192c919baff5793 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Fri, 21 Feb 2014 10:13:10 -0600 Subject: [PATCH 082/120] refer to non-deprecated config option in help memcache_servers was renamed to memcached_servers Change-Id: Ia04c6655f81d5b71cee0f16a7614262efa907b62 --- auth_token.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index dae51cb9..99f3dec3 100644 --- a/auth_token.py +++ b/auth_token.py @@ -268,13 +268,12 @@ opts = [ help='Directory used to cache files related to PKI tokens'), cfg.ListOpt('memcached_servers', deprecated_name='memcache_servers', - help='If defined, the memcache server(s) to use for' - ' caching'), + help='If defined, the memcache server(s) to use for caching'), cfg.IntOpt('token_cache_time', default=300, help='In order to prevent excessive requests and validations,' ' the middleware uses an in-memory cache for the tokens the' - ' Keystone API returns. This is only valid if memcache_servers' + ' Keystone API returns. This is only valid if memcached_servers' ' is defined. Set to -1 to disable caching completely.'), cfg.IntOpt('revocation_cache_time', default=1, From 4b2456a60208743eef48f87ff0f8480bd6bf8c55 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Fri, 21 Feb 2014 10:14:17 -0600 Subject: [PATCH 083/120] remove extra indentation Change-Id: I5b132e210b7155968a5d4f47da4ad8d7100d3e75 --- auth_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index dae51cb9..a6360d87 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1160,8 +1160,8 @@ class AuthProtocol(object): """ if self._cache: - self.LOG.debug('Storing %s token in memcache', token_id) - self._cache_store(token_id, (data, expires)) + self.LOG.debug('Storing %s token in memcache', token_id) + self._cache_store(token_id, (data, expires)) def _cache_store_invalid(self, token_id): """Store invalid token in cache.""" From 91ef21aced9ee553753be9db6ba46609c4c055eb Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Fri, 21 Feb 2014 10:39:05 -0600 Subject: [PATCH 084/120] correct typo of config option name in error message Change-Id: I146e6ac742649ec24c6fbfd8b73a5359e447e76f --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index dae51cb9..220ffee6 100644 --- a/auth_token.py +++ b/auth_token.py @@ -495,7 +495,7 @@ class AuthProtocol(object): raise ConfigurationError('memcache_security_strategy must be ' 'ENCRYPT or MAC') if not self._memcache_secret_key: - raise ConfigurationError('mecmache_secret_key must be defined ' + raise ConfigurationError('memcache_secret_key must be defined ' 'when a memcache_security_strategy ' 'is defined') From d367fbb8692d0cb50e9569836601c73b4f5268fb Mon Sep 17 00:00:00 2001 From: ZhiQiang Fan Date: Sat, 22 Feb 2014 12:20:24 +0800 Subject: [PATCH 085/120] Remove redundant default value None for dict.get The default value for dict.get is None, no need to specify again. Change-Id: I9f0204e59bd4c1642586bddfef0c75e0a7730925 --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index dae51cb9..809033e2 100644 --- a/auth_token.py +++ b/auth_token.py @@ -503,7 +503,7 @@ class AuthProtocol(object): cache = self._conf_get('cache') memcache_servers = self._conf_get('memcached_servers') - if cache and env.get(cache, None) is not None: + if cache and env.get(cache) is not None: # use the cache from the upstream filter self.LOG.info('Using %s memcache for caching token', cache) self._cache = env.get(cache) From 5709b884fadef9177b02caac53e027843f886b96 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Fri, 21 Feb 2014 16:48:37 -0600 Subject: [PATCH 086/120] demonstrate auth_token behavior with a simple echo service Change-Id: I522faed9fb9d338fcaf9aeec37e04fd5078b6306 --- auth_token.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/auth_token.py b/auth_token.py index dae51cb9..6ce2d096 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1418,3 +1418,30 @@ def app_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) return AuthProtocol(None, conf) + + +if __name__ == '__main__': + """Run this module directly to start a protected echo service:: + + $ python -m keystoneclient.middleware.auth_token + + When the ``auth_token`` module authenticates a request, the echo service + will respond with all the environment variables presented to it by this + module. + + """ + def echo_app(environ, start_response): + """A WSGI application that echoes the CGI environment to the user.""" + start_response('200 OK', [('Content-Type', 'application/json')]) + environment = dict((k, v) for k, v in six.iteritems(environ) + if k.startswith('HTTP_X_')) + yield jsonutils.dumps(environment) + + from wsgiref import simple_server + + # hardcode any non-default configuration here + conf = {'auth_protocol': 'http', 'admin_token': 'ADMIN'} + app = AuthProtocol(echo_app, conf) + server = simple_server.make_server('', 8000, app) + print('Serving on port 8000 (Ctrl+C to end)...') + server.serve_forever() From 61e9f9828553dd28f6dc3739c18488225361779a Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Thu, 20 Feb 2014 10:44:48 +1000 Subject: [PATCH 087/120] Use admin_prefix consistently The auth_admin_prefix config option is used manually in a number of places where it could just be set for all requests. Change-Id: Ib690d69bdf075cf7095841a6ab9f419dfe8b1ca8 --- auth_token.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/auth_token.py b/auth_token.py index a6360d87..eede9359 100644 --- a/auth_token.py +++ b/auth_token.py @@ -411,7 +411,7 @@ class AuthProtocol(object): auth_host = self._conf_get('auth_host') auth_port = int(self._conf_get('auth_port')) auth_protocol = self._conf_get('auth_protocol') - self.auth_admin_prefix = self._conf_get('auth_admin_prefix') + auth_admin_prefix = self._conf_get('auth_admin_prefix') self.auth_uri = self._conf_get('auth_uri') if netaddr.valid_ipv6(auth_host): @@ -432,6 +432,10 @@ class AuthProtocol(object): # documented in bug 1207517 self.auth_uri = self.request_uri + if auth_admin_prefix: + self.request_uri = "%s/%s" % (self.request_uri, + auth_admin_prefix.strip('/')) + # SSL self.cert_file = self._conf_get('certfile') self.key_file = self._conf_get('keyfile') @@ -762,8 +766,6 @@ class AuthProtocol(object): if body: kwargs['data'] = jsonutils.dumps(body) - path = self.auth_admin_prefix + path - response = self._http_request(method, path, **kwargs) try: @@ -1366,8 +1368,7 @@ class AuthProtocol(object): return self.cms_verify(data['signed']) def fetch_signing_cert(self): - path = self.auth_admin_prefix.rstrip('/') - path += '/v2.0/certificates/signing' + path = '/v2.0/certificates/signing' response = self._http_request('GET', path) def write_cert_file(data): @@ -1389,7 +1390,7 @@ class AuthProtocol(object): raise ServiceError('invalid json response') def fetch_ca_cert(self): - path = self.auth_admin_prefix.rstrip('/') + '/v2.0/certificates/ca' + path = '/v2.0/certificates/ca' response = self._http_request('GET', path) if response.status_code != 200: From 934969f816aa0b4058bf88d459f87b8e008450ad Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Thu, 27 Feb 2014 15:25:48 +1000 Subject: [PATCH 088/120] Rely on OSLO.config OSLO.config is now used by all the services. The hack in place to correctly find the right config file is apparently (by way of the comments) only required for the gate. Therefore we should be able to remove this. Change-Id: If5c5d5a4711de21624b2e75e99cc785520ab6aba --- auth_token.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/auth_token.py b/auth_token.py index a6360d87..ccab9546 100644 --- a/auth_token.py +++ b/auth_token.py @@ -152,6 +152,7 @@ import tempfile import time import netaddr +from oslo.config import cfg import six from six.moves import urllib @@ -164,23 +165,6 @@ from keystoneclient.openstack.common import timeutils from keystoneclient import utils -CONF = None -# to pass gate before oslo-config is deployed everywhere, -# try application copies first -for app in 'nova', 'glance', 'quantum', 'cinder': - try: - cfg = __import__('%s.openstack.common.cfg' % app, - fromlist=['%s.openstack.common' % app]) - # test which application middleware is running in - if hasattr(cfg, 'CONF') and 'config_file' in cfg.CONF: - CONF = cfg.CONF - break - except ImportError: - pass -if not CONF: - from oslo.config import cfg - CONF = cfg.CONF - # alternative middleware configuration in the main application's # configuration file e.g. in nova.conf # [keystone_authtoken] @@ -197,6 +181,7 @@ if not CONF: # 'swift.cache' key. However it could be different, depending on deployment. # To use Swift memcache, you must set the 'cache' option to the environment # key where the Swift cache object is stored. + opts = [ cfg.StrOpt('auth_admin_prefix', default='', @@ -310,6 +295,8 @@ opts = [ ' token binding is needed to be allowed. Finally the name of a' ' binding method that must be present in tokens.'), ] + +CONF = cfg.CONF CONF.register_opts(opts, group='keystone_authtoken') LIST_OF_VERSIONS_TO_ATTEMPT = ['v2.0', 'v3.0'] From 18c7c7480cf892109e975b897dec87d58b345550 Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Thu, 27 Feb 2014 15:29:25 +1000 Subject: [PATCH 089/120] Remove http_handler config option in auth_token This option was predominately used so that we could substitute a fake handler in place so that we could mock our HTTP calls. This has been replaced for some time now with httpretty so is no longer necessary. Change-Id: I8613f4e189ee977f17b9606547840d5e7847d77a --- auth_token.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/auth_token.py b/auth_token.py index a6360d87..0c7c00dd 100644 --- a/auth_token.py +++ b/auth_token.py @@ -233,12 +233,6 @@ opts = [ default=3, help='How many times are we trying to reconnect when' ' communicating with Identity API Server.'), - cfg.StrOpt('http_handler', - default=None, - help='Allows to pass in the name of a fake http_handler' - ' callback function used instead of httplib.HTTPConnection or' - ' httplib.HTTPSConnection. Useful for unit testing where' - ' network is not available.'), cfg.StrOpt('admin_token', secret=True, help='Single shared secret with the Keystone configuration' From 6c47dc7843c16fe137571b8ad78f5f7683d16e0b Mon Sep 17 00:00:00 2001 From: Adam Young Date: Fri, 28 Feb 2014 11:28:01 -0500 Subject: [PATCH 090/120] Atomic write of certificate files and revocation list Using a rename from a temporary file to avoid having partial writes. Closes-Bug: #1285833 Change-Id: I3e70c795be357ceab8e5a1d12202c04fd88a02b8 --- auth_token.py | 64 ++++++++++++++++++++------------------------------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/auth_token.py b/auth_token.py index 41de990e..427e7c5c 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1324,6 +1324,24 @@ class AuthProtocol(object): self.token_revocation_list = self.fetch_revocation_list() return self._token_revocation_list + def _atomic_write_to_signing_dir(self, file_name, value): + # In Python2, encoding is slow so the following check avoids it if it + # is not absolutely necessary. + if isinstance(value, six.text_type): + value = value.encode('utf-8') + + def _atomic_write(destination, data): + with tempfile.NamedTemporaryFile(dir=self.signing_dirname, + delete=False) as f: + f.write(data) + os.rename(f.name, destination) + + try: + _atomic_write(file_name, value) + except (OSError, IOError): + self.verify_signing_dir() + _atomic_write(file_name, value) + @token_revocation_list.setter def token_revocation_list(self, value): """Save a revocation list to memory and to disk. @@ -1333,15 +1351,7 @@ class AuthProtocol(object): """ self._token_revocation_list = jsonutils.loads(value) self.token_revocation_list_fetched_time = timeutils.utcnow() - - with tempfile.NamedTemporaryFile(dir=self.signing_dirname, - delete=False) as f: - # In Python2, encoding is slow so the following check avoids it if - # it is not absolutely necessary. - if isinstance(value, six.text_type): - value = value.encode('utf-8') - f.write(value) - os.rename(f.name, self.revoked_file_name) + self._atomic_write_to_signing_dir(self.revoked_file_name, value) def fetch_revocation_list(self, retry=True): headers = {'X-Auth-Token': self.get_admin_token()} @@ -1360,42 +1370,18 @@ class AuthProtocol(object): raise ServiceError('Revocation list improperly formatted.') return self.cms_verify(data['signed']) - def fetch_signing_cert(self): - path = '/v2.0/certificates/signing' + def _fetch_cert_file(self, cert_file_name, cert_type): + path = '/v2.0/certificates/' + cert_type response = self._http_request('GET', path) - - def write_cert_file(data): - with open(self.signing_cert_file_name, 'w') as certfile: - certfile.write(data) - if response.status_code != 200: raise exceptions.CertificateConfigError(response.text) + self._atomic_write_to_signing_dir(cert_file_name, response.text) - try: - try: - write_cert_file(response.text) - except IOError: - self.verify_signing_dir() - write_cert_file(response.text) - except (AssertionError, KeyError): - self.LOG.warn( - "Unexpected response from keystone service: %s", response.text) - raise ServiceError('invalid json response') + def fetch_signing_cert(self): + self._fetch_cert_file(self.signing_cert_file_name, 'signing') def fetch_ca_cert(self): - path = '/v2.0/certificates/ca' - response = self._http_request('GET', path) - - if response.status_code != 200: - raise exceptions.CertificateConfigError(response.text) - - try: - with open(self.signing_ca_file_name, 'w') as certfile: - certfile.write(response.text) - except (AssertionError, KeyError): - self.LOG.warn( - "Unexpected response from keystone service: %s", response.text) - raise ServiceError('invalid json response') + self._fetch_cert_file(self.signing_ca_file_name, 'ca') def filter_factory(global_conf, **local_conf): From 12785158764e837bef0bde554913bf209ec04725 Mon Sep 17 00:00:00 2001 From: Morgan Fainberg Date: Wed, 5 Mar 2014 15:15:15 -0800 Subject: [PATCH 091/120] Log the command output on CertificateConfigError In auth_token middleware if the CertificateConfigError is raise for a reason other than the certificate file missing or the ca cert file missing, log the actual output from the verify comment to aid in correcting the issue. Related-Bug: #1285833 Change-Id: I6b469b9037b7ef59993132f87a75152149ee6310 --- auth_token.py | 1 + 1 file changed, 1 insertion(+) diff --git a/auth_token.py b/auth_token.py index 427e7c5c..d6aac449 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1259,6 +1259,7 @@ class AuthProtocol(object): self.signing_ca_file_name): self.fetch_ca_cert() continue + self.LOG.error('CMS Verify output: %s', err.output) raise except cms.subprocess.CalledProcessError as err: self.LOG.warning('Verify error: %s', err) From c6dc4f546207910e314333a72cc35743389d3f32 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Thu, 6 Mar 2014 10:25:26 -0600 Subject: [PATCH 092/120] improve configuration help text in auth_token Related-Bug: 1287301 Change-Id: I3499fdf87b49b7499c525465ccd947edfa5cec6d --- auth_token.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/auth_token.py b/auth_token.py index 427e7c5c..9342eaca 100644 --- a/auth_token.py +++ b/auth_token.py @@ -262,16 +262,21 @@ opts = [ help='Directory used to cache files related to PKI tokens'), cfg.ListOpt('memcached_servers', deprecated_name='memcache_servers', - help='If defined, the memcache server(s) to use for caching'), + help='Optionally specify a list of memcached server(s) to' + ' use for caching. If left undefined, tokens will instead be' + ' cached in-process.'), cfg.IntOpt('token_cache_time', default=300, - help='In order to prevent excessive requests and validations,' - ' the middleware uses an in-memory cache for the tokens the' - ' Keystone API returns. This is only valid if memcached_servers' - ' is defined. Set to -1 to disable caching completely.'), + help='In order to prevent excessive effort spent validating' + ' tokens, the middleware caches previously-seen tokens for a' + ' configurable duration (in seconds). Set to -1 to disable' + ' caching completely.'), cfg.IntOpt('revocation_cache_time', default=1, - help='Value only used for unit testing'), + help='Determines the frequency at which the list of revoked' + ' tokens is retrieved from the Identity service (in seconds). A' + ' high number of revocation events combined with a low cache' + ' duration may significantly reduce performance.'), cfg.StrOpt('memcache_security_strategy', default=None, help='(optional) if defined, indicate whether token data' From 59cc918b3a22aca99db3c4b3ab1348adc54ee5e1 Mon Sep 17 00:00:00 2001 From: Joel Friedly Date: Mon, 3 Feb 2014 11:38:47 -0800 Subject: [PATCH 093/120] Make keystoneclient not log auth tokens I think I've looked at every log statement in this repo and I've either removed the tokens from the log strings or made them print out "" instead. Change-Id: I1efbc834fcab951f6797b56afb5c5519cc70d28c Closes-Bug: 1287938 --- auth_token.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/auth_token.py b/auth_token.py index 41de990e..60781aac 100644 --- a/auth_token.py +++ b/auth_token.py @@ -565,7 +565,7 @@ class AuthProtocol(object): versions.append(version['id']) except KeyError: self.LOG.error( - 'Invalid version response format from server', data) + 'Invalid version response format from server') raise ServiceError('Unable to parse version response ' 'from keystone') @@ -806,6 +806,7 @@ class AuthProtocol(object): "Unexpected response from keystone service: %s", data) raise ServiceError('invalid json response') except (ValueError): + data['access']['token']['id'] = '' self.LOG.warn( "Unable to parse expiration time from token: %s", data) raise ServiceError('invalid json response') @@ -838,13 +839,13 @@ class AuthProtocol(object): return data except NetworkError: self.LOG.debug('Token validation failure.', exc_info=True) - self.LOG.warn("Authorization failed for token %s", token_id) + self.LOG.warn("Authorization failed for token") raise InvalidUserToken('Token authorization failed') except Exception: self.LOG.debug('Token validation failure.', exc_info=True) if token_id: self._cache_store_invalid(token_id) - self.LOG.warn("Authorization failed for token %s", token_id) + self.LOG.warn("Authorization failed for token") raise InvalidUserToken('Token authorization failed') def _build_user_headers(self, token_info): @@ -1026,8 +1027,7 @@ class AuthProtocol(object): serialized = serialized.decode('utf-8') cached = jsonutils.loads(serialized) if cached == 'invalid': - self.LOG.debug('Cached Token %s is marked unauthorized', - token_id) + self.LOG.debug('Cached Token is marked unauthorized') raise InvalidUserToken('Token authorization failed') data, expires = cached @@ -1043,10 +1043,10 @@ class AuthProtocol(object): expires = timeutils.normalize_time(expires) utcnow = timeutils.utcnow() if ignore_expires or utcnow < expires: - self.LOG.debug('Returning cached token %s', token_id) + self.LOG.debug('Returning cached token') return data else: - self.LOG.debug('Cached Token %s seems expired', token_id) + self.LOG.debug('Cached Token seems expired') def _cache_store(self, token_id, data): """Store value into memcache. @@ -1155,14 +1155,14 @@ class AuthProtocol(object): """ if self._cache: - self.LOG.debug('Storing %s token in memcache', token_id) + self.LOG.debug('Storing token in memcache') self._cache_store(token_id, (data, expires)) def _cache_store_invalid(self, token_id): """Store invalid token in cache.""" if self._cache: self.LOG.debug( - 'Marking token %s as unauthorized in memcache', token_id) + 'Marking token as unauthorized in memcache') self._cache_store(token_id, 'invalid') def cert_file_missing(self, proc_output, file_name): @@ -1205,11 +1205,11 @@ class AuthProtocol(object): if response.status_code == 200: return data if response.status_code == 404: - self.LOG.warn("Authorization failed for token %s", user_token) + self.LOG.warn("Authorization failed for token") raise InvalidUserToken('Token authorization failed') if response.status_code == 401: self.LOG.info( - 'Keystone rejected admin token %s, resetting', headers) + 'Keystone rejected admin token, resetting') self.admin_token = None else: self.LOG.error('Bad response code while validating token: %s', @@ -1218,8 +1218,7 @@ class AuthProtocol(object): self.LOG.info('Retrying validation') return self._validate_user_token(user_token, env, False) else: - self.LOG.warn("Invalid user token: %s. Keystone response: %s.", - user_token, data) + self.LOG.warn("Invalid user token. Keystone response: %s", data) raise InvalidUserToken() @@ -1235,8 +1234,7 @@ class AuthProtocol(object): token_id = utils.hash_signed_token(signed_text) for revoked_id in revoked_ids: if token_id == revoked_id: - self.LOG.debug('Token %s is marked as having been revoked', - token_id) + self.LOG.debug('Token is marked as having been revoked') return True return False @@ -1350,8 +1348,7 @@ class AuthProtocol(object): if response.status_code == 401: if retry: self.LOG.info( - 'Keystone rejected admin token %s, resetting admin token', - headers) + 'Keystone rejected admin token, resetting admin token') self.admin_token = None return self.fetch_revocation_list(retry=False) if response.status_code != 200: From 687e33d62935d090a75c33d33c21a55d96aa4ad3 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Thu, 6 Mar 2014 10:26:58 -0600 Subject: [PATCH 094/120] increase default revocation_cache_time 1 second was originally intended as a useful value for testing purposes, but was never intended to be a production-friendly default. Raising the default value to match the default token_cache_time results in better production performance. Change-Id: I03939d41bc50b32a6aa5ebf93eb71103e16e988f Related-Bug: 1287301 --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 9342eaca..d2d307e3 100644 --- a/auth_token.py +++ b/auth_token.py @@ -272,7 +272,7 @@ opts = [ ' configurable duration (in seconds). Set to -1 to disable' ' caching completely.'), cfg.IntOpt('revocation_cache_time', - default=1, + default=300, help='Determines the frequency at which the list of revoked' ' tokens is retrieved from the Identity service (in seconds). A' ' high number of revocation events combined with a low cache' From c6e555b0481bba6cd4f88606b5a0ebf08d3a1743 Mon Sep 17 00:00:00 2001 From: Mouad Benchchaoui Date: Tue, 4 Mar 2014 16:11:58 +0100 Subject: [PATCH 095/120] Fix retry logic When passing retry=True to verify_uuid_token this later should re-call it self when the method fail and this time with retry=False. Change-Id: I7e8f478e5116ac4119df134587f29f3f25ae8a42 Closes-Bug: #1285847 --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 41de990e..2afe23d5 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1216,7 +1216,7 @@ class AuthProtocol(object): response.status_code) if retry: self.LOG.info('Retrying validation') - return self._validate_user_token(user_token, env, False) + return self.verify_uuid_token(user_token, False) else: self.LOG.warn("Invalid user token: %s. Keystone response: %s.", user_token, data) From 1b10289e921cb7b05b03d5f91faa80e320b9a0cc Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Thu, 20 Feb 2014 17:24:51 +1000 Subject: [PATCH 096/120] Use AccessInfo in auth_token middleware auth_token middleware parses the token and extracts the data manually. We have this parsing ability in AccessInfo now so we should use that instead. Change-Id: I23c6e33d327c3d3669fb977d9623b788324c4785 --- auth_token.py | 100 +++++++++++--------------------------------------- 1 file changed, 22 insertions(+), 78 deletions(-) diff --git a/auth_token.py b/auth_token.py index 83232097..15263e96 100644 --- a/auth_token.py +++ b/auth_token.py @@ -155,6 +155,7 @@ import netaddr import six from six.moves import urllib +from keystoneclient import access from keystoneclient.common import cms from keystoneclient import exceptions from keystoneclient.middleware import memcache_crypt @@ -863,96 +864,39 @@ class AuthProtocol(object): :raise InvalidUserToken when unable to parse token object """ - def get_tenant_info(): - """Returns a (tenant_id, tenant_name) tuple from context.""" - def essex(): - """Essex puts the tenant ID and name on the token.""" - return (token['tenant']['id'], token['tenant']['name']) - - def pre_diablo(): - """Pre-diablo, Keystone only provided tenantId.""" - return (token['tenantId'], token['tenantId']) - - def default_tenant(): - """Pre-grizzly, assume the user's default tenant.""" - return (user['tenantId'], user['tenantName']) - - for method in [essex, pre_diablo, default_tenant]: - try: - return method() - except KeyError: - pass + auth_ref = access.AccessInfo.factory(body=token_info) + roles = ",".join(auth_ref.role_names) + if _token_is_v2(token_info) and not auth_ref.project_id: raise InvalidUserToken('Unable to determine tenancy.') - # For clarity. set all those attributes that are optional in - # either a v2 or v3 token to None first - domain_id = None - domain_name = None - project_id = None - project_name = None - user_domain_id = None - user_domain_name = None - project_domain_id = None - project_domain_name = None - - if _token_is_v2(token_info): - user = token_info['access']['user'] - token = token_info['access']['token'] - roles = ','.join([role['name'] for role in user.get('roles', [])]) - catalog_root = token_info['access'] - catalog_key = 'serviceCatalog' - project_id, project_name = get_tenant_info() - else: - #v3 token - token = token_info['token'] - user = token['user'] - user_domain_id = user['domain']['id'] - user_domain_name = user['domain']['name'] - roles = (','.join([role['name'] - for role in token.get('roles', [])])) - catalog_root = token - catalog_key = 'catalog' - # For v3, the server will put in the default project if there is - # one, so no need for us to add it here (like we do for a v2 token) - if 'domain' in token: - domain_id = token['domain']['id'] - domain_name = token['domain']['name'] - elif 'project' in token: - project_id = token['project']['id'] - project_name = token['project']['name'] - project_domain_id = token['project']['domain']['id'] - project_domain_name = token['project']['domain']['name'] - - user_id = user['id'] - user_name = user['name'] - rval = { 'X-Identity-Status': 'Confirmed', - 'X-Domain-Id': domain_id, - 'X-Domain-Name': domain_name, - 'X-Project-Id': project_id, - 'X-Project-Name': project_name, - 'X-Project-Domain-Id': project_domain_id, - 'X-Project-Domain-Name': project_domain_name, - 'X-User-Id': user_id, - 'X-User-Name': user_name, - 'X-User-Domain-Id': user_domain_id, - 'X-User-Domain-Name': user_domain_name, + 'X-Domain-Id': auth_ref.domain_id, + 'X-Domain-Name': auth_ref.domain_name, + 'X-Project-Id': auth_ref.project_id, + 'X-Project-Name': auth_ref.project_name, + 'X-Project-Domain-Id': auth_ref.project_domain_id, + 'X-Project-Domain-Name': auth_ref.project_domain_name, + 'X-User-Id': auth_ref.user_id, + 'X-User-Name': auth_ref.username, + 'X-User-Domain-Id': auth_ref.user_domain_id, + 'X-User-Domain-Name': auth_ref.user_domain_name, 'X-Roles': roles, # Deprecated - 'X-User': user_name, - 'X-Tenant-Id': project_id, - 'X-Tenant-Name': project_name, - 'X-Tenant': project_name, + 'X-User': auth_ref.username, + 'X-Tenant-Id': auth_ref.project_id, + 'X-Tenant-Name': auth_ref.project_name, + 'X-Tenant': auth_ref.project_name, 'X-Role': roles, } self.LOG.debug("Received request from user: %s with project_id : %s" - " and roles: %s ", user_id, project_id, roles) + " and roles: %s ", + auth_ref.user_id, auth_ref.project_id, roles) - if self.include_service_catalog and catalog_key in catalog_root: - catalog = catalog_root[catalog_key] + if self.include_service_catalog and auth_ref.has_service_catalog(): + catalog = auth_ref.service_catalog.get_data() rval['X-Service-Catalog'] = jsonutils.dumps(catalog) return rval From c2286563ca24405ae59559d2882591b11ab7ab1c Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Sun, 16 Feb 2014 12:03:58 -0600 Subject: [PATCH 097/120] Fix doc build errors There were some parts that had invalid RST in their docstrings which caused warnings and errors to be generated. Related-Bug: #1278662 Change-Id: Ibb53e6f49b5fa100fa6ecfe47331f9a70729d03b --- auth_token.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index 83232097..1ac5a97d 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1180,9 +1180,9 @@ class AuthProtocol(object): :param retry: flag that forces the middleware to retry user authentication when an indeterminate response is received. Optional. - :return token object received from keystone on success - :raise InvalidUserToken if token is rejected - :raise ServiceError if unable to authenticate token + :return: token object received from keystone on success + :raise InvalidUserToken: if token is rejected + :raise ServiceError: if unable to authenticate token """ # Determine the highest api version we can use. From a15d0caddbe0e38def2d9e338c8e17e34ad60ebf Mon Sep 17 00:00:00 2001 From: wanghong Date: Fri, 21 Mar 2014 16:53:20 +0800 Subject: [PATCH 098/120] use v3 api to get certificates Let Token signing and ca certificates can be accessible at /v3/OS-SIMPLE-CERT/{ca,certificates}. Change-Id: I6c82d1f78ba1ff2ab110474623982542610b4d2d Closes-Bug: #1206345 --- auth_token.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 7bed4feb..aeb647ee 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1361,7 +1361,15 @@ class AuthProtocol(object): return self.cms_verify(data['signed']) def _fetch_cert_file(self, cert_file_name, cert_type): - path = '/v2.0/certificates/' + cert_type + if not self.auth_version: + self.auth_version = self._choose_api_version() + + if self.auth_version == 'v3.0': + if cert_type == 'signing': + cert_type = 'certificates' + path = '/v3/OS-SIMPLE-CERT/' + cert_type + else: + path = '/v2.0/certificates/' + cert_type response = self._http_request('GET', path) if response.status_code != 200: raise exceptions.CertificateConfigError(response.text) From 59a27edf30c205ce411910fcf59829b09968f010 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Mon, 17 Mar 2014 15:56:28 -0500 Subject: [PATCH 099/120] add pooling for cache references Change-Id: Iffb1d1bff5dc4437544a5aefef3bca0e5b17cc81 Closes-Bug: 1289074 --- auth_token.py | 71 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/auth_token.py b/auth_token.py index 7bed4feb..f3327450 100644 --- a/auth_token.py +++ b/auth_token.py @@ -143,6 +143,7 @@ keystone.token_info """ +import contextlib import datetime import logging import os @@ -450,9 +451,9 @@ class AuthProtocol(object): self.admin_password = self._conf_get('admin_password') self.admin_tenant_name = self._conf_get('admin_tenant_name') - # Token caching via memcache - self._cache = None - self._cache_initialized = False # cache already initialized? + # Token caching + self._cache_pool = None + self._cache_initialized = False # memcache value treatment, ENCRYPT or MAC self._memcache_security_strategy = \ self._conf_get('memcache_security_strategy') @@ -489,16 +490,9 @@ class AuthProtocol(object): 'is defined') def _init_cache(self, env): - cache = self._conf_get('cache') - memcache_servers = self._conf_get('memcached_servers') - - if cache and env.get(cache) is not None: - # use the cache from the upstream filter - self.LOG.info('Using %s memcache for caching token', cache) - self._cache = env.get(cache) - else: - # use Keystone memcache - self._cache = memorycache.get_client(memcache_servers) + self._cache_pool = CachePool( + env.get(self._conf_get('cache')), + self._conf_get('memcached_servers')) self._cache_initialized = True def _conf_get(self, name): @@ -980,10 +974,11 @@ class AuthProtocol(object): return token only if fresh (not expired). """ - if self._cache and token_id: + if token_id: if self._memcache_security_strategy is None: key = CACHE_KEY_TEMPLATE % token_id - serialized = self._cache.get(key) + with self._cache_pool.reserve() as cache: + serialized = cache.get(key) else: secret_key = self._memcache_secret_key if isinstance(secret_key, six.string_types): @@ -997,7 +992,8 @@ class AuthProtocol(object): security_strategy) cache_key = CACHE_KEY_TEMPLATE % ( memcache_crypt.get_cache_key(keys)) - raw_cached = self._cache.get(cache_key) + with self._cache_pool.reserve() as cache: + raw_cached = cache.get(cache_key) try: # unprotect_data will return None if raw_cached is None serialized = memcache_crypt.unprotect_data(keys, @@ -1064,9 +1060,8 @@ class AuthProtocol(object): cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) data_to_store = memcache_crypt.protect_data(keys, serialized_data) - self._cache.set(cache_key, - data_to_store, - time=self.token_cache_time) + with self._cache_pool.reserve() as cache: + cache.set(cache_key, data_to_store, time=self.token_cache_time) def _invalid_user_token(self, msg=False): # NOTE(jamielennox): use False as the default so that None is valid @@ -1146,16 +1141,13 @@ class AuthProtocol(object): quick check of token freshness on retrieval. """ - if self._cache: - self.LOG.debug('Storing token in memcache') - self._cache_store(token_id, (data, expires)) + self.LOG.debug('Storing token in cache') + self._cache_store(token_id, (data, expires)) def _cache_store_invalid(self, token_id): """Store invalid token in cache.""" - if self._cache: - self.LOG.debug( - 'Marking token as unauthorized in memcache') - self._cache_store(token_id, 'invalid') + self.LOG.debug('Marking token as unauthorized in cache') + self._cache_store(token_id, 'invalid') def cert_file_missing(self, proc_output, file_name): return (file_name in proc_output and not os.path.exists(file_name)) @@ -1374,6 +1366,33 @@ class AuthProtocol(object): self._fetch_cert_file(self.signing_ca_file_name, 'ca') +class CachePool(list): + """A lazy pool of cache references.""" + + def __init__(self, cache, memcached_servers): + self._environment_cache = cache + self._memcached_servers = memcached_servers + + @contextlib.contextmanager + def reserve(self): + """Context manager to manage a pooled cache reference.""" + if self._environment_cache is not None: + # skip pooling and just use the cache from the upstream filter + yield self._environment_cache + return # otherwise the context manager will continue! + + try: + c = self.pop() + except IndexError: + # the pool is empty, so we need to create a new client + c = memorycache.get_client(self._memcached_servers) + + try: + yield c + finally: + self.append(c) + + def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() From b9e475eed9851be2ee73963b4a39fadcf3f3efc7 Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Mon, 3 Mar 2014 13:12:22 +1000 Subject: [PATCH 100/120] Replace auth fragements with identity_uri The need for individual auth_host, auth_port etc variables for auth_token middleware made some sense when we were using httplib however as we have long moved on to requests they are no longer required and are a pain for configuration. DocImpact: auth_schema, auth_host, auth_port, auth_admin_prefix are all deprecated in favour of specifying the full url in the identity_uri property. Blueprint: identity-uri Change-Id: I1f8f5064ea8028af60f167df9b97e215cdadba44 --- auth_token.py | 74 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/auth_token.py b/auth_token.py index f3327450..358d815a 100644 --- a/auth_token.py +++ b/auth_token.py @@ -183,26 +183,44 @@ from keystoneclient import utils # To use Swift memcache, you must set the 'cache' option to the environment # key where the Swift cache object is stored. + +# NOTE(jamielennox): A number of options below are deprecated however are left +# in the list and only mentioned as deprecated in the help string. This is +# because we have to provide the same deprecation functionality for arguments +# passed in via the conf in __init__ (from paste) and there is no way to test +# that the default value was set or not in CONF. +# Also if we were to remove the options from the CONF list (as typical CONF +# deprecation works) then other projects will not be able to override the +# options via CONF. + opts = [ cfg.StrOpt('auth_admin_prefix', default='', - help='Prefix to prepend at the beginning of the path'), + help='Prefix to prepend at the beginning of the path. ' + 'Deprecated, use identity_uri.'), cfg.StrOpt('auth_host', default='127.0.0.1', - help='Host providing the admin Identity API endpoint'), + help='Host providing the admin Identity API endpoint. ' + 'Deprecated, use identity_uri.'), cfg.IntOpt('auth_port', default=35357, - help='Port of the admin Identity API endpoint'), + help='Port of the admin Identity API endpoint. ' + 'Deprecated, use identity_uri.'), cfg.StrOpt('auth_protocol', default='https', - help='Protocol of the admin Identity API endpoint' - '(http or https)'), + help='Protocol of the admin Identity API endpoint ' + '(http or https). Deprecated, use identity_uri.'), cfg.StrOpt('auth_uri', default=None, # FIXME(dolph): should be default='http://127.0.0.1:5000/v2.0/', # or (depending on client support) an unversioned, publicly # accessible identity endpoint (see bug 1207517) help='Complete public Identity API endpoint'), + cfg.StrOpt('identity_uri', + default=None, + help='Complete admin Identity API endpoint. This should ' + 'specify the unversioned root endpoint ' + 'eg. https://localhost:35357/'), cfg.StrOpt('auth_version', default=None, help='API version of the admin Identity API endpoint'), @@ -394,19 +412,34 @@ class AuthProtocol(object): (True, 'true', 't', '1', 'on', 'yes', 'y')) # where to find the auth service (we use this to validate tokens) - auth_host = self._conf_get('auth_host') - auth_port = int(self._conf_get('auth_port')) - auth_protocol = self._conf_get('auth_protocol') - auth_admin_prefix = self._conf_get('auth_admin_prefix') + self.request_uri = self._conf_get('identity_uri') self.auth_uri = self._conf_get('auth_uri') - if netaddr.valid_ipv6(auth_host): - # Note(dzyu) it is an IPv6 address, so it needs to be wrapped - # with '[]' to generate a valid IPv6 URL, based on - # http://www.ietf.org/rfc/rfc2732.txt - auth_host = '[%s]' % auth_host + # NOTE(jamielennox): it does appear here that our defaults arguments + # are backwards. We need to do it this way so that we can handle the + # same deprecation strategy for CONF and the conf variable. + if not self.request_uri: + self.LOG.warning("Configuring admin URI using auth fragments. " + "This is deprecated, use 'identity_uri' instead.") - self.request_uri = '%s://%s:%s' % (auth_protocol, auth_host, auth_port) + auth_host = self._conf_get('auth_host') + auth_port = int(self._conf_get('auth_port')) + auth_protocol = self._conf_get('auth_protocol') + auth_admin_prefix = self._conf_get('auth_admin_prefix') + + if netaddr.valid_ipv6(auth_host): + # Note(dzyu) it is an IPv6 address, so it needs to be wrapped + # with '[]' to generate a valid IPv6 URL, based on + # http://www.ietf.org/rfc/rfc2732.txt + auth_host = '[%s]' % auth_host + + self.request_uri = '%s://%s:%s' % (auth_protocol, auth_host, + auth_port) + if auth_admin_prefix: + self.request_uri = '%s/%s' % (self.request_uri, + auth_admin_prefix.strip('/')) + else: + self.request_uri = self.request_uri.rstrip('/') if self.auth_uri is None: self.LOG.warning( @@ -415,12 +448,11 @@ class AuthProtocol(object): 'authenticate against an admin endpoint') # FIXME(dolph): drop support for this fallback behavior as - # documented in bug 1207517 - self.auth_uri = self.request_uri - - if auth_admin_prefix: - self.request_uri = "%s/%s" % (self.request_uri, - auth_admin_prefix.strip('/')) + # documented in bug 1207517. + # NOTE(jamielennox): we urljoin '/' to get just the base URI as + # this is the original behaviour. + self.auth_uri = urllib.parse.urljoin(self.request_uri, '/') + self.auth_uri = self.auth_uri.rstrip('/') # SSL self.cert_file = self._conf_get('certfile') From 4ea85108229ed2b280cee48c7d62d14dff82640c Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Tue, 4 Mar 2014 12:02:56 +1000 Subject: [PATCH 101/120] Rename request_uri to identity_uri This makes it more consistent with the CONF options. Blueprint: identity-uri Change-Id: If4e32d232413e539b4c29035b253e9368b3fbd06 --- auth_token.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/auth_token.py b/auth_token.py index 358d815a..a0c65148 100644 --- a/auth_token.py +++ b/auth_token.py @@ -412,13 +412,13 @@ class AuthProtocol(object): (True, 'true', 't', '1', 'on', 'yes', 'y')) # where to find the auth service (we use this to validate tokens) - self.request_uri = self._conf_get('identity_uri') + self.identity_uri = self._conf_get('identity_uri') self.auth_uri = self._conf_get('auth_uri') # NOTE(jamielennox): it does appear here that our defaults arguments # are backwards. We need to do it this way so that we can handle the # same deprecation strategy for CONF and the conf variable. - if not self.request_uri: + if not self.identity_uri: self.LOG.warning("Configuring admin URI using auth fragments. " "This is deprecated, use 'identity_uri' instead.") @@ -433,13 +433,13 @@ class AuthProtocol(object): # http://www.ietf.org/rfc/rfc2732.txt auth_host = '[%s]' % auth_host - self.request_uri = '%s://%s:%s' % (auth_protocol, auth_host, - auth_port) + self.identity_uri = '%s://%s:%s' % (auth_protocol, auth_host, + auth_port) if auth_admin_prefix: - self.request_uri = '%s/%s' % (self.request_uri, - auth_admin_prefix.strip('/')) + self.identity_uri = '%s/%s' % (self.identity_uri, + auth_admin_prefix.strip('/')) else: - self.request_uri = self.request_uri.rstrip('/') + self.identity_uri = self.identity_uri.rstrip('/') if self.auth_uri is None: self.LOG.warning( @@ -451,7 +451,7 @@ class AuthProtocol(object): # documented in bug 1207517. # NOTE(jamielennox): we urljoin '/' to get just the base URI as # this is the original behaviour. - self.auth_uri = urllib.parse.urljoin(self.request_uri, '/') + self.auth_uri = urllib.parse.urljoin(self.identity_uri, '/') self.auth_uri = self.auth_uri.rstrip('/') # SSL @@ -722,7 +722,7 @@ class AuthProtocol(object): :raise ServerError when unable to communicate with keystone """ - url = "%s/%s" % (self.request_uri, path.lstrip('/')) + url = "%s/%s" % (self.identity_uri, path.lstrip('/')) kwargs.setdefault('timeout', self.http_connect_timeout) if self.cert_file and self.key_file: From 84d94e33f4441ea0525b88d15ff5108777e8e122 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Sun, 30 Mar 2014 09:42:55 -0500 Subject: [PATCH 102/120] Prefer () to continue line per PEP8 There were some long lines that were split using \ rather than (). PEP8 recommends using () -- http://legacy.python.org/dev/peps/pep-0008/#maximum-line-length Change-Id: I8e140e507d0d9991094be13ebafa7fc700c1a02e --- auth_token.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/auth_token.py b/auth_token.py index b1a8515e..ad99a032 100644 --- a/auth_token.py +++ b/auth_token.py @@ -488,13 +488,13 @@ class AuthProtocol(object): self._cache_pool = None self._cache_initialized = False # memcache value treatment, ENCRYPT or MAC - self._memcache_security_strategy = \ - self._conf_get('memcache_security_strategy') + self._memcache_security_strategy = ( + self._conf_get('memcache_security_strategy')) if self._memcache_security_strategy is not None: - self._memcache_security_strategy = \ - self._memcache_security_strategy.upper() - self._memcache_secret_key = \ - self._conf_get('memcache_secret_key') + self._memcache_security_strategy = ( + self._memcache_security_strategy.upper()) + self._memcache_secret_key = ( + self._conf_get('memcache_secret_key')) self._assert_valid_memcache_protection_config() # By default the token will be cached for 5 minutes self.token_cache_time = int(self._conf_get('token_cache_time')) @@ -506,8 +506,8 @@ class AuthProtocol(object): self.http_connect_timeout = (http_connect_timeout_cfg and int(http_connect_timeout_cfg)) self.auth_version = None - self.http_request_max_retries = \ - self._conf_get('http_request_max_retries') + self.http_request_max_retries = ( + self._conf_get('http_request_max_retries')) self.include_service_catalog = self._conf_get( 'include_service_catalog') From 2fbd5089d39b0a7756476f7096d57dcb4477f624 Mon Sep 17 00:00:00 2001 From: Dolph Mathews Date: Wed, 9 Apr 2014 08:09:09 -0500 Subject: [PATCH 103/120] eliminate race condition fetching certs There's a race between the time that a CertificateConfigError is raised, and when we check to see what caused it. Eliminating the checks and unconditionally fetching certificates eliminates the race. Giant thanks to Jamie Lennox for identifying the root cause described above! Co-Authored-By: David Stanek Change-Id: I19113496ceaecdc03e209d550e0db156df95f9b8 Closes-Bug: 1285833 --- auth_token.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/auth_token.py b/auth_token.py index b1a8515e..9c5a709e 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1125,9 +1125,6 @@ class AuthProtocol(object): self.LOG.debug('Marking token as unauthorized in cache') self._cache_store(token_id, 'invalid') - def cert_file_missing(self, proc_output, file_name): - return (file_name in proc_output and not os.path.exists(file_name)) - def verify_uuid_token(self, user_token, retry=True): """Authenticate user token with keystone. @@ -1201,28 +1198,32 @@ class AuthProtocol(object): def cms_verify(self, data): """Verifies the signature of the provided data's IAW CMS syntax. - If either of the certificate files are missing, fetch them and + If either of the certificate files might be missing, fetch them and retry. """ - while True: + def verify(): try: - output = cms.cms_verify(data, self.signing_cert_file_name, - self.signing_ca_file_name) - except exceptions.CertificateConfigError as err: - if self.cert_file_missing(err.output, - self.signing_cert_file_name): - self.fetch_signing_cert() - continue - if self.cert_file_missing(err.output, - self.signing_ca_file_name): - self.fetch_ca_cert() - continue - self.LOG.error('CMS Verify output: %s', err.output) - raise + return cms.cms_verify(data, self.signing_cert_file_name, + self.signing_ca_file_name) except cms.subprocess.CalledProcessError as err: self.LOG.warning('Verify error: %s', err) raise - return output + + try: + return verify() + except exceptions.CertificateConfigError: + # the certs might be missing; unconditionally fetch to avoid racing + self.fetch_signing_cert() + self.fetch_ca_cert() + + try: + # retry with certs in place + return verify() + except exceptions.CertificateConfigError as err: + # if this is still occurring, something else is wrong and we + # need err.output to identify the problem + self.LOG.error('CMS Verify output: %s', err.output) + raise def verify_signed_token(self, signed_text): """Check that the token is unrevoked and has a valid signature.""" From d8ec5d35de4cb8b9fa770fb1a827677f1c4c273b Mon Sep 17 00:00:00 2001 From: mathrock Date: Sat, 12 Apr 2014 00:45:27 -0400 Subject: [PATCH 104/120] Fix typo of ANS1 to ASN1 Replace all occurrences of 'ANS1|ans1' with 'ASN1|asn1'. Keep cms.is_ans1_token() around for backwards compatibility. Change-Id: I89da78b89aa9daf2637754dc93031d7ca81e85cb Closes-bug: 1306874 --- auth_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 9c5a709e..4594ffb3 100644 --- a/auth_token.py +++ b/auth_token.py @@ -847,7 +847,7 @@ class AuthProtocol(object): cached = self._cache_get(token_id) if cached: return cached - if cms.is_ans1_token(user_token): + if cms.is_asn1_token(user_token): verified = self.verify_signed_token(user_token) data = jsonutils.loads(verified) else: From 4fb30601d71055d34ca355f906f99deb6ea13cfb Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Sat, 12 Apr 2014 11:41:44 -0500 Subject: [PATCH 105/120] Deprecate admin_token option in auth_token The admin_token option shouldn't be used with the auth_token middleware. It's used to specify a token to be used to perform operations on the identity server, so would typically be set to the admin token. The admin token should only be used to initially set up the Keystone server, and then the admin token functionality should be disabled. If this recommended setup is used then the auth_token middleware shouldn't be using the admin token / auth_token. In preparing for removal of the admin_token option, the option is now deprecated. A warning will be logged if it's set. DocImpact Change-Id: I5bc4f4a6ad7984892151c8011ccd92f166aba4c2 Closes-Bug: #1306981 --- auth_token.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index dbee45b1..b02d1181 100644 --- a/auth_token.py +++ b/auth_token.py @@ -240,9 +240,12 @@ opts = [ ' communicating with Identity API Server.'), cfg.StrOpt('admin_token', secret=True, - help='Single shared secret with the Keystone configuration' + help='This option is deprecated and may be removed in a future' + ' release. Single shared secret with the Keystone configuration' ' used for bootstrapping a Keystone installation, or otherwise' - ' bypassing the normal authentication process.'), + ' bypassing the normal authentication process. This option' + ' should not be used, use `admin_user` and `admin_password`' + ' instead.'), cfg.StrOpt('admin_user', help='Keystone account username'), cfg.StrOpt('admin_password', @@ -479,6 +482,12 @@ class AuthProtocol(object): # Credentials used to verify this component with the Auth service since # validating tokens is a privileged call self.admin_token = self._conf_get('admin_token') + if self.admin_token: + self.LOG.warning( + "The admin_token option in the auth_token middleware is " + "deprecated and should not be used. The admin_user and " + "admin_password options should be used instead. The " + "admin_token option may be removed in a future release.") self.admin_token_expiry = None self.admin_user = self._conf_get('admin_user') self.admin_password = self._conf_get('admin_password') From 13703bf5dd86c1aaaff93312e4cba0d3f5883fb0 Mon Sep 17 00:00:00 2001 From: Adam Young Date: Mon, 21 Apr 2014 16:49:09 -0400 Subject: [PATCH 106/120] replace double quotes with single. Change-Id: Ib2c828525fe3bafac8ed2f402a477ba62bbf6471 --- auth_token.py | 57 ++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/auth_token.py b/auth_token.py index dbee45b1..702015ff 100644 --- a/auth_token.py +++ b/auth_token.py @@ -420,8 +420,9 @@ class AuthProtocol(object): # are backwards. We need to do it this way so that we can handle the # same deprecation strategy for CONF and the conf variable. if not self.identity_uri: - self.LOG.warning("Configuring admin URI using auth fragments. " - "This is deprecated, use 'identity_uri' instead.") + self.LOG.warning('Configuring admin URI using auth fragments. ' + 'This is deprecated, use \'identity_uri\'' + ' instead.') auth_host = self._conf_get('auth_host') auth_port = int(self._conf_get('auth_port')) @@ -572,8 +573,8 @@ class AuthProtocol(object): versions = [] response, data = self._json_request('GET', '/') if response.status_code == 501: - self.LOG.warning("Old keystone installation found...assuming v2.0") - versions.append("v2.0") + self.LOG.warning('Old keystone installation found...assuming v2.0') + versions.append('v2.0') elif response.status_code != 300: self.LOG.error('Unable to get version info from keystone: %s', response.status_code) @@ -675,9 +676,9 @@ class AuthProtocol(object): return token else: if not self.delay_auth_decision: - self.LOG.warn("Unable to find authentication token" - " in headers") - self.LOG.debug("Headers: %s", env) + self.LOG.warn('Unable to find authentication token' + ' in headers') + self.LOG.debug('Headers: %s', env) raise InvalidUserToken('Unable to find token in headers') def _reject_request(self, env, start_response): @@ -723,7 +724,7 @@ class AuthProtocol(object): :raise ServerError when unable to communicate with keystone """ - url = "%s/%s" % (self.identity_uri, path.lstrip('/')) + url = '%s/%s' % (self.identity_uri, path.lstrip('/')) kwargs.setdefault('timeout', self.http_connect_timeout) if self.cert_file and self.key_file: @@ -822,12 +823,12 @@ class AuthProtocol(object): return (token, timeutils.normalize_time(datetime_expiry)) except (AssertionError, KeyError): self.LOG.warn( - "Unexpected response from keystone service: %s", data) + 'Unexpected response from keystone service: %s', data) raise ServiceError('invalid json response') except (ValueError): data['access']['token']['id'] = '' self.LOG.warn( - "Unable to parse expiration time from token: %s", data) + 'Unable to parse expiration time from token: %s', data) raise ServiceError('invalid json response') def _validate_user_token(self, user_token, env, retry=True): @@ -858,13 +859,13 @@ class AuthProtocol(object): return data except NetworkError: self.LOG.debug('Token validation failure.', exc_info=True) - self.LOG.warn("Authorization failed for token") + self.LOG.warn('Authorization failed for token') raise InvalidUserToken('Token authorization failed') except Exception: self.LOG.debug('Token validation failure.', exc_info=True) if token_id: self._cache_store_invalid(token_id) - self.LOG.warn("Authorization failed for token") + self.LOG.warn('Authorization failed for token') raise InvalidUserToken('Token authorization failed') def _build_user_headers(self, token_info): @@ -878,7 +879,7 @@ class AuthProtocol(object): """ auth_ref = access.AccessInfo.factory(body=token_info) - roles = ",".join(auth_ref.role_names) + roles = ','.join(auth_ref.role_names) if _token_is_v2(token_info) and not auth_ref.project_id: raise InvalidUserToken('Unable to determine tenancy.') @@ -904,8 +905,8 @@ class AuthProtocol(object): 'X-Role': roles, } - self.LOG.debug("Received request from user: %s with project_id : %s" - " and roles: %s ", + self.LOG.debug('Received request from user: %s with project_id : %s' + ' and roles: %s ', auth_ref.user_id, auth_ref.project_id, roles) if self.include_service_catalog and auth_ref.has_service_catalog(): @@ -1070,7 +1071,7 @@ class AuthProtocol(object): # no bind provided and none required return else: - self.LOG.info("No bind information present in token.") + self.LOG.info('No bind information present in token.') self._invalid_user_token() # get the named mode if bind_mode is not one of the predefined @@ -1080,32 +1081,32 @@ class AuthProtocol(object): name = bind_mode if name and name not in bind: - self.LOG.info("Named bind mode %s not in bind information", name) + self.LOG.info('Named bind mode %s not in bind information', name) self._invalid_user_token() for bind_type, identifier in six.iteritems(bind): if bind_type == BIND_MODE.KERBEROS: if not env.get('AUTH_TYPE', '').lower() == 'negotiate': - self.LOG.info("Kerberos credentials required and " - "not present.") + self.LOG.info('Kerberos credentials required and ' + 'not present.') self._invalid_user_token() if not env.get('REMOTE_USER') == identifier: - self.LOG.info("Kerberos credentials do not match " - "those in bind.") + self.LOG.info('Kerberos credentials do not match ' + 'those in bind.') self._invalid_user_token() - self.LOG.debug("Kerberos bind authentication successful.") + self.LOG.debug('Kerberos bind authentication successful.') elif bind_mode == BIND_MODE.PERMISSIVE: - self.LOG.debug("Ignoring Unknown bind for permissive mode: " - "%(bind_type)s: %(identifier)s.", + self.LOG.debug('Ignoring Unknown bind for permissive mode: ' + '%(bind_type)s: %(identifier)s.', {'bind_type': bind_type, 'identifier': identifier}) else: - self.LOG.info("Couldn't verify unknown bind: %(bind_type)s: " - "%(identifier)s.", + self.LOG.info('Couldn`t verify unknown bind: %(bind_type)s: ' + '%(identifier)s.', {'bind_type': bind_type, 'identifier': identifier}) self._invalid_user_token() @@ -1162,7 +1163,7 @@ class AuthProtocol(object): if response.status_code == 200: return data if response.status_code == 404: - self.LOG.warn("Authorization failed for token") + self.LOG.warn('Authorization failed for token') raise InvalidUserToken('Token authorization failed') if response.status_code == 401: self.LOG.info( @@ -1175,7 +1176,7 @@ class AuthProtocol(object): self.LOG.info('Retrying validation') return self.verify_uuid_token(user_token, False) else: - self.LOG.warn("Invalid user token. Keystone response: %s", data) + self.LOG.warn('Invalid user token. Keystone response: %s', data) raise InvalidUserToken() From 9f846d1f7166c87f00ba11b55c2cd89be2e7dbdd Mon Sep 17 00:00:00 2001 From: Adam Young Date: Mon, 10 Mar 2014 15:12:15 -0400 Subject: [PATCH 107/120] remove universal_newlines Need to make sure that binary and text are both handled correctly for cms calls. Blueprint: compress-tokens Change-Id: If3ed5f339b53942d4ed6d6b2d9fc4eebd7180b0a --- auth_token.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 702015ff..d92610fb 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1204,8 +1204,9 @@ class AuthProtocol(object): """ def verify(): try: - return cms.cms_verify(data, self.signing_cert_file_name, - self.signing_ca_file_name) + return cms.cms_verify( + data, self.signing_cert_file_name, + self.signing_ca_file_name).decode('utf-8') except cms.subprocess.CalledProcessError as err: self.LOG.warning('Verify error: %s', err) raise From 5df3897ac5103cc1bdbcc0bccd3f1e3d724808a9 Mon Sep 17 00:00:00 2001 From: Alexei Kornienko Date: Wed, 5 Mar 2014 16:50:37 +0200 Subject: [PATCH 108/120] Ensure that cached token is not revoked We need to ensure that tokens won't stay in cache after they have been revoked. Changed default revocation_cache_time 300 -> 10 seconds. revocation_cache_time has to be << than token_cache_time to make token cache efficient. Fixes bug #1287301 Change-Id: I14c0eacac3b431c06e40385c891a6636736e5b4a --- auth_token.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/auth_token.py b/auth_token.py index 702015ff..95be453f 100644 --- a/auth_token.py +++ b/auth_token.py @@ -277,7 +277,7 @@ opts = [ ' configurable duration (in seconds). Set to -1 to disable' ' caching completely.'), cfg.IntOpt('revocation_cache_time', - default=300, + default=10, help='Determines the frequency at which the list of revoked' ' tokens is retrieved from the Identity service (in seconds). A' ' high number of revocation events combined with a low cache' @@ -832,7 +832,7 @@ class AuthProtocol(object): raise ServiceError('invalid json response') def _validate_user_token(self, user_token, env, retry=True): - """Authenticate user using PKI + """Authenticate user token :param user_token: user's token id :param retry: Ignored, as it is not longer relevant @@ -847,12 +847,17 @@ class AuthProtocol(object): token_id = cms.cms_hash_token(user_token) cached = self._cache_get(token_id) if cached: - return cached - if cms.is_asn1_token(user_token): + data = cached + elif cms.is_asn1_token(user_token): verified = self.verify_signed_token(user_token) data = jsonutils.loads(verified) else: data = self.verify_uuid_token(user_token, retry) + # A token stored in Memcached might have been revoked + # regardless of initial mechanism used to validate it, + # and needs to be checked. + if self._is_token_id_in_revoked_list(token_id): + raise InvalidUserToken('Token authorization failed') expires = confirm_token_not_expired(data) self._confirm_token_bind(data, env) self._cache_put(token_id, data, expires) @@ -1182,19 +1187,20 @@ class AuthProtocol(object): def is_signed_token_revoked(self, signed_text): """Indicate whether the token appears in the revocation list.""" - revocation_list = self.token_revocation_list - revoked_tokens = revocation_list.get('revoked', []) - if not revoked_tokens: - return - revoked_ids = (x['id'] for x in revoked_tokens) if isinstance(signed_text, six.text_type): signed_text = signed_text.encode('utf-8') token_id = utils.hash_signed_token(signed_text) - for revoked_id in revoked_ids: - if token_id == revoked_id: - self.LOG.debug('Token is marked as having been revoked') - return True - return False + return self._is_token_id_in_revoked_list(token_id) + + def _is_token_id_in_revoked_list(self, token_id): + """Indicate whether the token_id appears in the revocation list.""" + revocation_list = self.token_revocation_list + revoked_tokens = revocation_list.get('revoked', None) + if not revoked_tokens: + return False + + revoked_ids = (x['id'] for x in revoked_tokens) + return token_id in revoked_ids def cms_verify(self, data): """Verifies the signature of the provided data's IAW CMS syntax. From e505df6fee73623de0cdad9a913298dd0b677d19 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Tue, 22 Apr 2014 16:06:26 -0500 Subject: [PATCH 109/120] Debug log when token found in revocation list The auth_token middleware didn't log when a token is rejected because it's in the revocation list. This adds a log message so that it's easier to debug problems. Change-Id: I1388ed04641d209ba2083a1096488edc22267ebe --- auth_token.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/auth_token.py b/auth_token.py index 95be453f..ce8aab42 100644 --- a/auth_token.py +++ b/auth_token.py @@ -857,6 +857,7 @@ class AuthProtocol(object): # regardless of initial mechanism used to validate it, # and needs to be checked. if self._is_token_id_in_revoked_list(token_id): + self.LOG.debug('Token is marked as having been revoked') raise InvalidUserToken('Token authorization failed') expires = confirm_token_not_expired(data) self._confirm_token_bind(data, env) @@ -1190,7 +1191,10 @@ class AuthProtocol(object): if isinstance(signed_text, six.text_type): signed_text = signed_text.encode('utf-8') token_id = utils.hash_signed_token(signed_text) - return self._is_token_id_in_revoked_list(token_id) + is_revoked = self._is_token_id_in_revoked_list(token_id) + if is_revoked: + self.LOG.debug('Token is marked as having been revoked') + return is_revoked def _is_token_id_in_revoked_list(self, token_id): """Indicate whether the token_id appears in the revocation list.""" From 964a92e18cdf373579ce08e024d76e133003aa98 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Thu, 1 May 2014 14:35:20 -0500 Subject: [PATCH 110/120] auth_token configurable check of revocations for cached The auth_token middleware would fail if it couldn't fetch the revocation list. If the system is configured for UUID tokens then the revocation list may not be available. With this fix, the revocation list will only be checked for cached tokens if the new check_revocations_for_cached option is set to True. Also, this change prevents the revocation list from being checked twice for a PKI token that's validate off-line. Change-Id: I5408bbe12aefda608ebcb81cf3c7ef068b2bf2f6 Closes-Bug: #1312858 --- auth_token.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/auth_token.py b/auth_token.py index 90e7dde5..cfe75a0e 100644 --- a/auth_token.py +++ b/auth_token.py @@ -315,6 +315,10 @@ opts = [ ' unknown the token will be rejected. "required" any form of' ' token binding is needed to be allowed. Finally the name of a' ' binding method that must be present in tokens.'), + cfg.BoolOpt('check_revocations_for_cached', default=False, + help='If true, the revocation list will be checked for cached' + ' tokens. This requires that PKI tokens are configured on the' + ' Keystone server.'), ] CONF = cfg.CONF @@ -522,6 +526,9 @@ class AuthProtocol(object): self.include_service_catalog = self._conf_get( 'include_service_catalog') + self.check_revocations_for_cached = self._conf_get( + 'check_revocations_for_cached') + def _assert_valid_memcache_protection_config(self): if self._memcache_security_strategy: if self._memcache_security_strategy not in ('MAC', 'ENCRYPT'): @@ -857,17 +864,23 @@ class AuthProtocol(object): cached = self._cache_get(token_id) if cached: data = cached + + if self.check_revocations_for_cached: + # A token stored in Memcached might have been revoked + # regardless of initial mechanism used to validate it, + # and needs to be checked. + is_revoked = self._is_token_id_in_revoked_list(token_id) + if is_revoked: + self.LOG.debug( + 'Token is marked as having been revoked') + raise InvalidUserToken( + 'Token authorization failed') + elif cms.is_asn1_token(user_token): verified = self.verify_signed_token(user_token) data = jsonutils.loads(verified) else: data = self.verify_uuid_token(user_token, retry) - # A token stored in Memcached might have been revoked - # regardless of initial mechanism used to validate it, - # and needs to be checked. - if self._is_token_id_in_revoked_list(token_id): - self.LOG.debug('Token is marked as having been revoked') - raise InvalidUserToken('Token authorization failed') expires = confirm_token_not_expired(data) self._confirm_token_bind(data, env) self._cache_put(token_id, data, expires) From a9910a0e0833a6817fea694dd0cd9b4674fc3039 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Fri, 2 May 2014 16:12:20 +0200 Subject: [PATCH 111/120] fixed typos found by RETF rules rules are avaialble at https://en.wikipedia.org/wiki/Wikipedia:AutoWikiBrowser/Typos Change-Id: I67fb3e0d02c931cb7e605ac74ea8272956afa8e1 --- auth_token.py | 2 +- s3_token.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/auth_token.py b/auth_token.py index 90e7dde5..a2949faa 100644 --- a/auth_token.py +++ b/auth_token.py @@ -221,7 +221,7 @@ opts = [ default=None, help='Complete admin Identity API endpoint. This should ' 'specify the unversioned root endpoint ' - 'eg. https://localhost:35357/'), + 'e.g. https://localhost:35357/'), cfg.StrOpt('auth_version', default=None, help='API version of the admin Identity API endpoint'), diff --git a/s3_token.py b/s3_token.py index 5cbf6a72..0d080d3e 100644 --- a/s3_token.py +++ b/s3_token.py @@ -61,7 +61,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): :param rest_with_last: If True, trailing data will be returned as part of last segment. If False, and there is trailing data, raises ValueError. - :returns: list of segments with a length of maxsegs (non-existant + :returns: list of segments with a length of maxsegs (non-existent segments will return as None) :raises: ValueError if given an invalid path """ @@ -183,7 +183,7 @@ class S3Token(object): token = req.headers.get('X-Auth-Token', req.headers.get('X-Storage-Token')) if not token: - msg = 'You did not specify a auth or a storage token. skipping.' + msg = 'You did not specify an auth or a storage token. skipping.' self.logger.debug(msg) return self.app(environ, start_response) From f15de7768dd7239f77e1b6647d6c0dfa3a2fb3b0 Mon Sep 17 00:00:00 2001 From: Jamie Lennox Date: Tue, 22 Apr 2014 12:17:42 +1000 Subject: [PATCH 112/120] Make auth_token return a V2 Catalog As there is no way to distinguish a v2 or v3 catalog from the headers provided to an application we will for the meantime always return a v2 catalog. This should not cause any issues as the full token data is not provided to the service so there is no-one that will get caught out by a v2/v3 mix, and anyone that is already supporting the v3 catalog format will have to support the v2 catalog format as well so it will continue to work. Change-Id: Ic9b38e0ba4682b47ae295bd3f89bac59ef7437cf Closes-Bug: #1302970 --- auth_token.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/auth_token.py b/auth_token.py index 637b3b26..25544ddd 100644 --- a/auth_token.py +++ b/auth_token.py @@ -108,6 +108,8 @@ HTTP_X_ROLES HTTP_X_SERVICE_CATALOG json encoded keystone service catalog (optional). + For compatibility reasons this catalog will always be in the V2 catalog + format even if it is a v3 token. HTTP_X_TENANT_ID *Deprecated* in favor of HTTP_X_PROJECT_ID @@ -371,6 +373,42 @@ def confirm_token_not_expired(data): return timeutils.isotime(at=expires, subsecond=True) +def _v3_to_v2_catalog(catalog): + """Convert a catalog to v2 format. + + X_SERVICE_CATALOG must be specified in v2 format. If you get a token + that is in v3 convert it. + """ + v2_services = [] + for v3_service in catalog: + # first copy over the entries we allow for the service + v2_service = {'type': v3_service['type']} + try: + v2_service['name'] = v3_service['name'] + except KeyError: + pass + + # now convert the endpoints. Because in v3 we specify region per + # URL not per group we have to collect all the entries of the same + # region together before adding it to the new service. + regions = {} + for v3_endpoint in v3_service.get('endpoints', []): + region_name = v3_endpoint.get('region') + try: + region = regions[region_name] + except KeyError: + region = {'region': region_name} if region_name else {} + regions[region_name] = region + + interface_name = v3_endpoint['interface'].lower() + 'URL' + region[interface_name] = v3_endpoint['url'] + + v2_service['endpoints'] = list(regions.values()) + v2_services.append(v2_service) + + return v2_services + + def safe_quote(s): """URL-encode strings that are not already URL-encoded.""" return urllib.parse.quote(s) if s == urllib.parse.unquote(s) else s @@ -939,6 +977,8 @@ class AuthProtocol(object): if self.include_service_catalog and auth_ref.has_service_catalog(): catalog = auth_ref.service_catalog.get_data() + if _token_is_v3(token_info): + catalog = _v3_to_v2_catalog(catalog) rval['X-Service-Catalog'] = jsonutils.dumps(catalog) return rval From 2248cff5d79d3560aa14afaffd410c908f489831 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Mon, 5 May 2014 17:09:23 -0500 Subject: [PATCH 113/120] Cached tokens aren't expired The auth_token test said that the cached tokens are expired. The tokens weren't expired, so remove the code. Change-Id: I8ce30cc09ee9bbc19cc4ebdb5d935a80d2d5d473 --- auth_token.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auth_token.py b/auth_token.py index 637b3b26..72575a8f 100644 --- a/auth_token.py +++ b/auth_token.py @@ -972,7 +972,7 @@ class AuthProtocol(object): env_key = self._header_to_env_var(key) return env.get(env_key, default) - def _cache_get(self, token_id, ignore_expires=False): + def _cache_get(self, token_id): """Return token information from cache. If token is invalid raise InvalidUserToken @@ -1035,7 +1035,7 @@ class AuthProtocol(object): expires = timeutils.normalize_time(expires) utcnow = timeutils.utcnow() - if ignore_expires or utcnow < expires: + if utcnow < expires: self.LOG.debug('Returning cached token') return data else: From 799c4448b435dc7e50f8513c2db149d23c4266cd Mon Sep 17 00:00:00 2001 From: Adam Young Date: Tue, 4 Feb 2014 20:43:07 -0500 Subject: [PATCH 114/120] Compressed Signature and Validation Allows for a new form of document signature. pkiz_sign will take data and encode it in a string that starts with the substring "PKIZ_". This prefix indicates that the data has been: 1) Signed via PKI in Crypto Message Syntax (CMS) in binary (DER) format 2) Compressed using zlib (comparable to gzip) 3) urlsafe-base64 decoded This process is reversed to validate the data. middleware/auth_token.py will be capable of validating Keystone tokens that are marshalled in the new format. The current existing "PKI" tokens will continue to be identified with "MII", issued by default, and validated as well. It will require corresponding changes on the Keystone server to issue the new token format. A separate script for generating the sample data used in the unit tests, examples/pki/gen_cmsz.py, also serves as an example of how to call the API from Python code. Some of the sample data for the old tests had to be regenerated. A stray comma in one of the JSON files made for non-parsing JSON. Blueprint: compress-tokens Closes-Bug: #1255321 Change-Id: Ia9a66ba3742da0bcd58c4c096b28cc8a66ad6569 --- auth_token.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/auth_token.py b/auth_token.py index 637b3b26..f11a2602 100644 --- a/auth_token.py +++ b/auth_token.py @@ -875,7 +875,9 @@ class AuthProtocol(object): 'Token is marked as having been revoked') raise InvalidUserToken( 'Token authorization failed') - + elif cms.is_pkiz(user_token): + verified = self.verify_pkiz_token(user_token) + data = jsonutils.loads(verified) elif cms.is_asn1_token(user_token): verified = self.verify_signed_token(user_token) data = jsonutils.loads(verified) @@ -1228,7 +1230,7 @@ class AuthProtocol(object): revoked_ids = (x['id'] for x in revoked_tokens) return token_id in revoked_ids - def cms_verify(self, data): + def cms_verify(self, data, inform=cms.PKI_ASN1_FORM): """Verifies the signature of the provided data's IAW CMS syntax. If either of the certificate files might be missing, fetch them and @@ -1236,9 +1238,9 @@ class AuthProtocol(object): """ def verify(): try: - return cms.cms_verify( - data, self.signing_cert_file_name, - self.signing_ca_file_name).decode('utf-8') + return cms.cms_verify(data, self.signing_cert_file_name, + self.signing_ca_file_name, + inform=inform).decode('utf-8') except cms.subprocess.CalledProcessError as err: self.LOG.warning('Verify error: %s', err) raise @@ -1265,7 +1267,19 @@ class AuthProtocol(object): raise InvalidUserToken('Token has been revoked') formatted = cms.token_to_cms(signed_text) - return self.cms_verify(formatted) + verified = self.cms_verify(formatted) + return verified + + def verify_pkiz_token(self, signed_text): + if self.is_signed_token_revoked(signed_text): + raise InvalidUserToken('Token has been revoked') + try: + uncompressed = cms.pkiz_uncompress(signed_text) + verified = self.cms_verify(uncompressed, inform=cms.PKIZ_CMS_FORM) + return verified + # TypeError If the signed_text is not zlib compressed + except TypeError: + raise InvalidUserToken(signed_text) def verify_signing_dir(self): if os.path.exists(self.signing_dirname): From 877aac75744cc4cf1d86d68e4cebe7e0dda3f24b Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Tue, 6 May 2014 19:33:46 -0500 Subject: [PATCH 115/120] auth_token hashes PKI token once auth_token was hashing the PKI token multiple times. With this change, the token is hashed once. Change-Id: I70d3339d09deb2d3528f141d37138971038f4075 Related-Bug: #1174499 --- auth_token.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/auth_token.py b/auth_token.py index f11a2602..24785751 100644 --- a/auth_token.py +++ b/auth_token.py @@ -164,7 +164,6 @@ from keystoneclient.middleware import memcache_crypt from keystoneclient.openstack.common import jsonutils from keystoneclient.openstack.common import memorycache from keystoneclient.openstack.common import timeutils -from keystoneclient import utils # alternative middleware configuration in the main application's @@ -876,10 +875,10 @@ class AuthProtocol(object): raise InvalidUserToken( 'Token authorization failed') elif cms.is_pkiz(user_token): - verified = self.verify_pkiz_token(user_token) + verified = self.verify_pkiz_token(user_token, token_id) data = jsonutils.loads(verified) elif cms.is_asn1_token(user_token): - verified = self.verify_signed_token(user_token) + verified = self.verify_signed_token(user_token, token_id) data = jsonutils.loads(verified) else: data = self.verify_uuid_token(user_token, retry) @@ -1210,11 +1209,8 @@ class AuthProtocol(object): raise InvalidUserToken() - def is_signed_token_revoked(self, signed_text): + def is_signed_token_revoked(self, token_id): """Indicate whether the token appears in the revocation list.""" - if isinstance(signed_text, six.text_type): - signed_text = signed_text.encode('utf-8') - token_id = utils.hash_signed_token(signed_text) is_revoked = self._is_token_id_in_revoked_list(token_id) if is_revoked: self.LOG.debug('Token is marked as having been revoked') @@ -1261,17 +1257,17 @@ class AuthProtocol(object): self.LOG.error('CMS Verify output: %s', err.output) raise - def verify_signed_token(self, signed_text): + def verify_signed_token(self, signed_text, token_id): """Check that the token is unrevoked and has a valid signature.""" - if self.is_signed_token_revoked(signed_text): + if self.is_signed_token_revoked(token_id): raise InvalidUserToken('Token has been revoked') formatted = cms.token_to_cms(signed_text) verified = self.cms_verify(formatted) return verified - def verify_pkiz_token(self, signed_text): - if self.is_signed_token_revoked(signed_text): + def verify_pkiz_token(self, signed_text, token_id): + if self.is_signed_token_revoked(token_id): raise InvalidUserToken('Token has been revoked') try: uncompressed = cms.pkiz_uncompress(signed_text) From 4f9ffa3199f1dab9cf043c70b2a957f56d72472c Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Mon, 19 May 2014 17:13:01 +0200 Subject: [PATCH 116/120] replace string format arguments with function parameters There are files containing string format arguments inside logging messages. Using logging function parameters should be preferred. Change-Id: Ibd9def4cf111d5dcf15dff64f85a723214a3c14e Closes-Bug: #1320930 --- s3_token.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/s3_token.py b/s3_token.py index 0d080d3e..786da5cf 100644 --- a/s3_token.py +++ b/s3_token.py @@ -102,7 +102,7 @@ class S3Token(object): """Common initialization code.""" self.app = app self.logger = logging.getLogger(conf.get('log_name', __name__)) - self.logger.debug('Starting the %s component' % PROTOCOL_NAME) + self.logger.debug('Starting the %s component', PROTOCOL_NAME) self.reseller_prefix = conf.get('reseller_prefix', 'AUTH_') # where to find the auth service (we use this to validate tokens) @@ -149,13 +149,13 @@ class S3Token(object): headers=headers, data=creds_json, verify=self.verify) except requests.exceptions.RequestException as e: - self.logger.info('HTTP connection exception: %s' % e) + self.logger.info('HTTP connection exception: %s', e) resp = self.deny_request('InvalidURI') raise ServiceError(resp) if response.status_code < 200 or response.status_code >= 300: - self.logger.debug('Keystone reply error: status=%s reason=%s' % - (response.status_code, response.reason)) + self.logger.debug('Keystone reply error: status=%s reason=%s', + response.status_code, response.reason) resp = self.deny_request('AccessDenied') raise ServiceError(resp) @@ -192,7 +192,7 @@ class S3Token(object): access, signature = auth_header.split(' ')[-1].rsplit(':', 1) except ValueError: msg = 'You have an invalid Authorization header: %s' - self.logger.debug(msg % (auth_header)) + self.logger.debug(msg, auth_header) return self.deny_request('InvalidURI')(environ, start_response) # NOTE(chmou): This is to handle the special case with nova @@ -215,7 +215,7 @@ class S3Token(object): 'token': token, 'signature': signature}} creds_json = jsonutils.dumps(creds) - self.logger.debug('Connecting to Keystone sending this JSON: %s' % + self.logger.debug('Connecting to Keystone sending this JSON: %s', creds_json) # NOTE(vish): We could save a call to keystone by having # keystone return token, tenant, user, and roles @@ -230,11 +230,11 @@ class S3Token(object): except ServiceError as e: resp = e.args[0] msg = 'Received error, exiting middleware with error: %s' - self.logger.debug(msg % (resp.status_code)) + self.logger.debug(msg, resp.status_code) return resp(environ, start_response) - self.logger.debug('Keystone Reply: Status: %d, Output: %s' % ( - resp.status_code, resp.content)) + self.logger.debug('Keystone Reply: Status: %d, Output: %s', + resp.status_code, resp.content) try: identity_info = resp.json() @@ -242,12 +242,12 @@ class S3Token(object): tenant = identity_info['access']['token']['tenant'] except (ValueError, KeyError): error = 'Error on keystone reply: %d %s' - self.logger.debug(error % (resp.status_code, str(resp.content))) + self.logger.debug(error, resp.status_code, resp.content) return self.deny_request('InvalidURI')(environ, start_response) req.headers['X-Auth-Token'] = token_id tenant_to_connect = force_tenant or tenant['id'] - self.logger.debug('Connecting with tenant: %s' % (tenant_to_connect)) + self.logger.debug('Connecting with tenant: %s', tenant_to_connect) new_tenant_name = '%s%s' % (self.reseller_prefix, tenant_to_connect) environ['PATH_INFO'] = environ['PATH_INFO'].replace(account, new_tenant_name) From 3778b8d66cdf6de2e4dae1eef248ab27bac18243 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Tue, 6 May 2014 19:36:59 -0500 Subject: [PATCH 117/120] auth_token middleware hashes tokens with configurable algorithm The auth_token middleware always hashed PKI Tokens with MD5. This change makes it so that PKI tokens can be hashed with SHA256 or any other algorithm supported by hashlib.new(). This is for security hardening. auth_token has a new config option 'hash_algorithms' that is set to the list of algorithms that will be used for hashing PKI tokens. This will typically be set to a single hash algorithm which must match the hash algorithm set in Keystone. Otherwise the tokens in the revocation list will not match, leading to revoked tokens being still usable. During a transition from one algorithm to another, 'hash_algorithms' is set to both the new algorithm and the old algorithm. Both of the hash algorithms will be used to match against the revocation list and cache. Once the tokens using the old algorithm have expired the old algorithm can be removed from the list. 'hash_algorithms' defaults to ['md5'] for backwards compatibility. DocImpact SecurityImpact Closes-Bug: #1174499 Change-Id: Ie524125dc5f6f1076bfd47db3a414b178e4dac80 --- auth_token.py | 83 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/auth_token.py b/auth_token.py index d2ff4e95..5fbbe7f6 100644 --- a/auth_token.py +++ b/auth_token.py @@ -320,6 +320,16 @@ opts = [ help='If true, the revocation list will be checked for cached' ' tokens. This requires that PKI tokens are configured on the' ' Keystone server.'), + cfg.ListOpt('hash_algorithms', default=['md5'], + help='Hash algorithms to use for hashing PKI tokens. This may' + ' be a single algorithm or multiple. The algorithms are those' + ' supported by Python standard hashlib.new(). The hashes will' + ' be tried in the order given, so put the preferred one first' + ' for performance. The result of the first hash will be stored' + ' in the cache. This will typically be set to multiple values' + ' only while migrating from a less secure algorithm to a more' + ' secure one. Once all the old tokens are expired this option' + ' should be set to a single value for better performance.'), ] CONF = cfg.CONF @@ -897,8 +907,8 @@ class AuthProtocol(object): token_id = None try: - token_id = cms.cms_hash_token(user_token) - cached = self._cache_get(token_id) + token_ids, cached = self._check_user_token_cached(user_token) + token_id = token_ids[0] if cached: data = cached @@ -906,17 +916,18 @@ class AuthProtocol(object): # A token stored in Memcached might have been revoked # regardless of initial mechanism used to validate it, # and needs to be checked. - is_revoked = self._is_token_id_in_revoked_list(token_id) - if is_revoked: - self.LOG.debug( - 'Token is marked as having been revoked') - raise InvalidUserToken( - 'Token authorization failed') + for tid in token_ids: + is_revoked = self._is_token_id_in_revoked_list(tid) + if is_revoked: + self.LOG.debug( + 'Token is marked as having been revoked') + raise InvalidUserToken( + 'Token authorization failed') elif cms.is_pkiz(user_token): - verified = self.verify_pkiz_token(user_token, token_id) + verified = self.verify_pkiz_token(user_token, token_ids) data = jsonutils.loads(verified) elif cms.is_asn1_token(user_token): - verified = self.verify_signed_token(user_token, token_id) + verified = self.verify_signed_token(user_token, token_ids) data = jsonutils.loads(verified) else: data = self.verify_uuid_token(user_token, retry) @@ -935,6 +946,39 @@ class AuthProtocol(object): self.LOG.warn('Authorization failed for token') raise InvalidUserToken('Token authorization failed') + def _check_user_token_cached(self, user_token): + """Check if the token is cached already. + + Returns a tuple. The first element is a list of token IDs, where the + first one is the preferred hash. + + The second element is the token data from the cache if the token was + cached, otherwise ``None``. + + :raises InvalidUserToken: if the token is invalid + + """ + + if cms.is_asn1_token(user_token): + # user_token is a PKI token that's not hashed. + + algos = self._conf_get('hash_algorithms') + token_hashes = list(cms.cms_hash_token(user_token, mode=algo) + for algo in algos) + + for token_hash in token_hashes: + cached = self._cache_get(token_hash) + if cached: + return (token_hashes, cached) + + # The token wasn't found using any hash algorithm. + return (token_hashes, None) + + # user_token is either a UUID token or a hashed PKI token. + token_id = user_token + cached = self._cache_get(token_id) + return ([token_id], cached) + def _build_user_headers(self, token_info): """Convert token object into headers. @@ -1249,12 +1293,13 @@ class AuthProtocol(object): raise InvalidUserToken() - def is_signed_token_revoked(self, token_id): + def is_signed_token_revoked(self, token_ids): """Indicate whether the token appears in the revocation list.""" - is_revoked = self._is_token_id_in_revoked_list(token_id) - if is_revoked: - self.LOG.debug('Token is marked as having been revoked') - return is_revoked + for token_id in token_ids: + if self._is_token_id_in_revoked_list(token_id): + self.LOG.debug('Token is marked as having been revoked') + return True + return False def _is_token_id_in_revoked_list(self, token_id): """Indicate whether the token_id appears in the revocation list.""" @@ -1297,17 +1342,17 @@ class AuthProtocol(object): self.LOG.error('CMS Verify output: %s', err.output) raise - def verify_signed_token(self, signed_text, token_id): + def verify_signed_token(self, signed_text, token_ids): """Check that the token is unrevoked and has a valid signature.""" - if self.is_signed_token_revoked(token_id): + if self.is_signed_token_revoked(token_ids): raise InvalidUserToken('Token has been revoked') formatted = cms.token_to_cms(signed_text) verified = self.cms_verify(formatted) return verified - def verify_pkiz_token(self, signed_text, token_id): - if self.is_signed_token_revoked(token_id): + def verify_pkiz_token(self, signed_text, token_ids): + if self.is_signed_token_revoked(token_ids): raise InvalidUserToken('Token has been revoked') try: uncompressed = cms.pkiz_uncompress(signed_text) From 30efde6dfd7933c4c5e02c3d8b3ca9d4701552fd Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Sun, 1 Jun 2014 10:45:50 -0500 Subject: [PATCH 118/120] Refactor auth_token token cache members to class The token cache members are moved from AuthToken to their own class. Change-Id: Ibf00d39435fa7a6d9a92a9bdfacc3f1b07f890ef --- auth_token.py | 398 +++++++++++++++++++++++++++----------------------- 1 file changed, 219 insertions(+), 179 deletions(-) diff --git a/auth_token.py b/auth_token.py index 5fbbe7f6..d6fe3dd8 100644 --- a/auth_token.py +++ b/auth_token.py @@ -545,20 +545,18 @@ class AuthProtocol(object): self.admin_password = self._conf_get('admin_password') self.admin_tenant_name = self._conf_get('admin_tenant_name') - # Token caching - self._cache_pool = None - self._cache_initialized = False - # memcache value treatment, ENCRYPT or MAC - self._memcache_security_strategy = ( + memcache_security_strategy = ( self._conf_get('memcache_security_strategy')) - if self._memcache_security_strategy is not None: - self._memcache_security_strategy = ( - self._memcache_security_strategy.upper()) - self._memcache_secret_key = ( - self._conf_get('memcache_secret_key')) - self._assert_valid_memcache_protection_config() - # By default the token will be cached for 5 minutes - self.token_cache_time = int(self._conf_get('token_cache_time')) + + self._token_cache = TokenCache( + self.LOG, + cache_time=int(self._conf_get('token_cache_time')), + hash_algorithms=self._conf_get('hash_algorithms'), + env_cache_name=self._conf_get('cache'), + memcached_servers=self._conf_get('memcached_servers'), + memcache_security_strategy=memcache_security_strategy, + memcache_secret_key=self._conf_get('memcache_secret_key')) + self._token_revocation_list = None self._token_revocation_list_fetched_time = None self.token_revocation_list_cache_timeout = datetime.timedelta( @@ -576,22 +574,6 @@ class AuthProtocol(object): self.check_revocations_for_cached = self._conf_get( 'check_revocations_for_cached') - def _assert_valid_memcache_protection_config(self): - if self._memcache_security_strategy: - if self._memcache_security_strategy not in ('MAC', 'ENCRYPT'): - raise ConfigurationError('memcache_security_strategy must be ' - 'ENCRYPT or MAC') - if not self._memcache_secret_key: - raise ConfigurationError('memcache_secret_key must be defined ' - 'when a memcache_security_strategy ' - 'is defined') - - def _init_cache(self, env): - self._cache_pool = CachePool( - env.get(self._conf_get('cache')), - self._conf_get('memcached_servers')) - self._cache_initialized = True - def _conf_get(self, name): # try config from paste-deploy first if name in self.conf: @@ -665,9 +647,7 @@ class AuthProtocol(object): """ self.LOG.debug('Authenticating user token') - # initialize memcache if we haven't done so - if not self._cache_initialized: - self._init_cache(env) + self._token_cache.initialize(env) try: self._remove_auth_headers(env) @@ -907,7 +887,7 @@ class AuthProtocol(object): token_id = None try: - token_ids, cached = self._check_user_token_cached(user_token) + token_ids, cached = self._token_cache.get(user_token) token_id = token_ids[0] if cached: data = cached @@ -933,7 +913,7 @@ class AuthProtocol(object): data = self.verify_uuid_token(user_token, retry) expires = confirm_token_not_expired(data) self._confirm_token_bind(data, env) - self._cache_put(token_id, data, expires) + self._token_cache.store(token_id, data, expires) return data except NetworkError: self.LOG.debug('Token validation failure.', exc_info=True) @@ -942,43 +922,10 @@ class AuthProtocol(object): except Exception: self.LOG.debug('Token validation failure.', exc_info=True) if token_id: - self._cache_store_invalid(token_id) + self._token_cache.store_invalid(token_id) self.LOG.warn('Authorization failed for token') raise InvalidUserToken('Token authorization failed') - def _check_user_token_cached(self, user_token): - """Check if the token is cached already. - - Returns a tuple. The first element is a list of token IDs, where the - first one is the preferred hash. - - The second element is the token data from the cache if the token was - cached, otherwise ``None``. - - :raises InvalidUserToken: if the token is invalid - - """ - - if cms.is_asn1_token(user_token): - # user_token is a PKI token that's not hashed. - - algos = self._conf_get('hash_algorithms') - token_hashes = list(cms.cms_hash_token(user_token, mode=algo) - for algo in algos) - - for token_hash in token_hashes: - cached = self._cache_get(token_hash) - if cached: - return (token_hashes, cached) - - # The token wasn't found using any hash algorithm. - return (token_hashes, None) - - # user_token is either a UUID token or a hashed PKI token. - token_id = user_token - cached = self._cache_get(token_id) - return ([token_id], cached) - def _build_user_headers(self, token_info): """Convert token object into headers. @@ -1057,102 +1004,6 @@ class AuthProtocol(object): env_key = self._header_to_env_var(key) return env.get(env_key, default) - def _cache_get(self, token_id): - """Return token information from cache. - - If token is invalid raise InvalidUserToken - return token only if fresh (not expired). - """ - - if token_id: - if self._memcache_security_strategy is None: - key = CACHE_KEY_TEMPLATE % token_id - with self._cache_pool.reserve() as cache: - serialized = cache.get(key) - else: - secret_key = self._memcache_secret_key - if isinstance(secret_key, six.string_types): - secret_key = secret_key.encode('utf-8') - security_strategy = self._memcache_security_strategy - if isinstance(security_strategy, six.string_types): - security_strategy = security_strategy.encode('utf-8') - keys = memcache_crypt.derive_keys( - token_id, - secret_key, - security_strategy) - cache_key = CACHE_KEY_TEMPLATE % ( - memcache_crypt.get_cache_key(keys)) - with self._cache_pool.reserve() as cache: - raw_cached = cache.get(cache_key) - try: - # unprotect_data will return None if raw_cached is None - serialized = memcache_crypt.unprotect_data(keys, - raw_cached) - except Exception: - msg = 'Failed to decrypt/verify cache data' - self.LOG.exception(msg) - # this should have the same effect as data not - # found in cache - serialized = None - - if serialized is None: - return None - - # Note that 'invalid' and (data, expires) are the only - # valid types of serialized cache entries, so there is not - # a collision with jsonutils.loads(serialized) == None. - if not isinstance(serialized, six.string_types): - serialized = serialized.decode('utf-8') - cached = jsonutils.loads(serialized) - if cached == 'invalid': - self.LOG.debug('Cached Token is marked unauthorized') - raise InvalidUserToken('Token authorization failed') - - data, expires = cached - - try: - expires = timeutils.parse_isotime(expires) - except ValueError: - # Gracefully handle upgrade of expiration times from *nix - # timestamps to ISO 8601 formatted dates by ignoring old cached - # values. - return - - expires = timeutils.normalize_time(expires) - utcnow = timeutils.utcnow() - if utcnow < expires: - self.LOG.debug('Returning cached token') - return data - else: - self.LOG.debug('Cached Token seems expired') - - def _cache_store(self, token_id, data): - """Store value into memcache. - - data may be the string 'invalid' or a tuple like (data, expires) - - """ - serialized_data = jsonutils.dumps(data) - if isinstance(serialized_data, six.text_type): - serialized_data = serialized_data.encode('utf-8') - if self._memcache_security_strategy is None: - cache_key = CACHE_KEY_TEMPLATE % token_id - data_to_store = serialized_data - else: - secret_key = self._memcache_secret_key - if isinstance(secret_key, six.string_types): - secret_key = secret_key.encode('utf-8') - security_strategy = self._memcache_security_strategy - if isinstance(security_strategy, six.string_types): - security_strategy = security_strategy.encode('utf-8') - keys = memcache_crypt.derive_keys( - token_id, secret_key, security_strategy) - cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) - data_to_store = memcache_crypt.protect_data(keys, serialized_data) - - with self._cache_pool.reserve() as cache: - cache.set(cache_key, data_to_store, time=self.token_cache_time) - def _invalid_user_token(self, msg=False): # NOTE(jamielennox): use False as the default so that None is valid if msg is False: @@ -1224,21 +1075,6 @@ class AuthProtocol(object): 'identifier': identifier}) self._invalid_user_token() - def _cache_put(self, token_id, data, expires): - """Put token data into the cache. - - Stores the parsed expire date in cache allowing - quick check of token freshness on retrieval. - - """ - self.LOG.debug('Storing token in cache') - self._cache_store(token_id, (data, expires)) - - def _cache_store_invalid(self, token_id): - """Store invalid token in cache.""" - self.LOG.debug('Marking token as unauthorized in cache') - self._cache_store(token_id, 'invalid') - def verify_uuid_token(self, user_token, retry=True): """Authenticate user token with keystone. @@ -1507,6 +1343,210 @@ class CachePool(list): self.append(c) +class TokenCache(object): + """Encapsulates the auth_token token cache functionality. + + auth_token caches tokens that it's seen so that when a token is re-used the + middleware doesn't have to do a more expensive operation (like going to the + identity server) to validate the token. + + initialize() must be called before calling the other methods. + + Store a valid token in the cache using store(); mark a token as invalid in + the cache using store_invalid(). + + Check if a token is in the cache and retrieve it using get(). + + """ + + _INVALID_INDICATOR = 'invalid' + + def __init__(self, log, cache_time=None, hash_algorithms=None, + env_cache_name=None, memcached_servers=None, + memcache_security_strategy=None, memcache_secret_key=None): + self.LOG = log + self._cache_time = cache_time + self._hash_algorithms = hash_algorithms + self._env_cache_name = env_cache_name + self._memcached_servers = memcached_servers + + # memcache value treatment, ENCRYPT or MAC + self._memcache_security_strategy = memcache_security_strategy + if self._memcache_security_strategy is not None: + self._memcache_security_strategy = ( + self._memcache_security_strategy.upper()) + self._memcache_secret_key = memcache_secret_key + + self._cache_pool = None + self._initialized = False + + self._assert_valid_memcache_protection_config() + + def initialize(self, env): + if self._initialized: + return + + self._cache_pool = CachePool(env.get(self._env_cache_name), + self._memcached_servers) + self._initialized = True + + def get(self, user_token): + """Check if the token is cached already. + + Returns a tuple. The first element is a list of token IDs, where the + first one is the preferred hash. + + The second element is the token data from the cache if the token was + cached, otherwise ``None``. + + :raises InvalidUserToken: if the token is invalid + + """ + + if cms.is_asn1_token(user_token): + # user_token is a PKI token that's not hashed. + + token_hashes = list(cms.cms_hash_token(user_token, mode=algo) + for algo in self._hash_algorithms) + + for token_hash in token_hashes: + cached = self._cache_get(token_hash) + if cached: + return (token_hashes, cached) + + # The token wasn't found using any hash algorithm. + return (token_hashes, None) + + # user_token is either a UUID token or a hashed PKI token. + token_id = user_token + cached = self._cache_get(token_id) + return ([token_id], cached) + + def store(self, token_id, data, expires): + """Put token data into the cache. + + Stores the parsed expire date in cache allowing + quick check of token freshness on retrieval. + + """ + self.LOG.debug('Storing token in cache') + self._cache_store(token_id, (data, expires)) + + def store_invalid(self, token_id): + """Store invalid token in cache.""" + self.LOG.debug('Marking token as unauthorized in cache') + self._cache_store(token_id, self._INVALID_INDICATOR) + + def _assert_valid_memcache_protection_config(self): + if self._memcache_security_strategy: + if self._memcache_security_strategy not in ('MAC', 'ENCRYPT'): + raise ConfigurationError('memcache_security_strategy must be ' + 'ENCRYPT or MAC') + if not self._memcache_secret_key: + raise ConfigurationError('memcache_secret_key must be defined ' + 'when a memcache_security_strategy ' + 'is defined') + + def _cache_get(self, token_id): + """Return token information from cache. + + If token is invalid raise InvalidUserToken + return token only if fresh (not expired). + """ + + if not token_id: + # Nothing to do + return + + if self._memcache_security_strategy is None: + key = CACHE_KEY_TEMPLATE % token_id + with self._cache_pool.reserve() as cache: + serialized = cache.get(key) + else: + secret_key = self._memcache_secret_key + if isinstance(secret_key, six.string_types): + secret_key = secret_key.encode('utf-8') + security_strategy = self._memcache_security_strategy + if isinstance(security_strategy, six.string_types): + security_strategy = security_strategy.encode('utf-8') + keys = memcache_crypt.derive_keys( + token_id, + secret_key, + security_strategy) + cache_key = CACHE_KEY_TEMPLATE % ( + memcache_crypt.get_cache_key(keys)) + with self._cache_pool.reserve() as cache: + raw_cached = cache.get(cache_key) + try: + # unprotect_data will return None if raw_cached is None + serialized = memcache_crypt.unprotect_data(keys, + raw_cached) + except Exception: + msg = 'Failed to decrypt/verify cache data' + self.LOG.exception(msg) + # this should have the same effect as data not + # found in cache + serialized = None + + if serialized is None: + return None + + # Note that _INVALID_INDICATOR and (data, expires) are the only + # valid types of serialized cache entries, so there is not + # a collision with jsonutils.loads(serialized) == None. + if not isinstance(serialized, six.string_types): + serialized = serialized.decode('utf-8') + cached = jsonutils.loads(serialized) + if cached == self._INVALID_INDICATOR: + self.LOG.debug('Cached Token is marked unauthorized') + raise InvalidUserToken('Token authorization failed') + + data, expires = cached + + try: + expires = timeutils.parse_isotime(expires) + except ValueError: + # Gracefully handle upgrade of expiration times from *nix + # timestamps to ISO 8601 formatted dates by ignoring old cached + # values. + return + + expires = timeutils.normalize_time(expires) + utcnow = timeutils.utcnow() + if utcnow < expires: + self.LOG.debug('Returning cached token') + return data + else: + self.LOG.debug('Cached Token seems expired') + + def _cache_store(self, token_id, data): + """Store value into memcache. + + data may be _INVALID_INDICATOR or a tuple like (data, expires) + + """ + serialized_data = jsonutils.dumps(data) + if isinstance(serialized_data, six.text_type): + serialized_data = serialized_data.encode('utf-8') + if self._memcache_security_strategy is None: + cache_key = CACHE_KEY_TEMPLATE % token_id + data_to_store = serialized_data + else: + secret_key = self._memcache_secret_key + if isinstance(secret_key, six.string_types): + secret_key = secret_key.encode('utf-8') + security_strategy = self._memcache_security_strategy + if isinstance(security_strategy, six.string_types): + security_strategy = security_strategy.encode('utf-8') + keys = memcache_crypt.derive_keys( + token_id, secret_key, security_strategy) + cache_key = CACHE_KEY_TEMPLATE % memcache_crypt.get_cache_key(keys) + data_to_store = memcache_crypt.protect_data(keys, serialized_data) + + with self._cache_pool.reserve() as cache: + cache.set(cache_key, data_to_store, time=self._cache_time) + + def filter_factory(global_conf, **local_conf): """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() From 389f90461fd276911665f935d19536b7588e6813 Mon Sep 17 00:00:00 2001 From: Brant Knudson Date: Fri, 30 May 2014 10:02:51 -0500 Subject: [PATCH 119/120] auth_token _cache_get checks token expired When auth_token stores the token in the cache, it's stored with the expiration time; but when the token is retrieved from the cache, if the expiration time has passed the token is treated as if it wasn't cached. This creates extra work because now auth_token has to check the token expiration (either by decrypting the PKI token or online validation for UUID tokens). With this change, getting the token from the cache will fail if the expiration is past. Change-Id: Id0ec6b3c2e5af4a2d910f16da4e0312733fc2198 --- auth_token.py | 1 + 1 file changed, 1 insertion(+) diff --git a/auth_token.py b/auth_token.py index d6fe3dd8..593518b2 100644 --- a/auth_token.py +++ b/auth_token.py @@ -1518,6 +1518,7 @@ class TokenCache(object): return data else: self.LOG.debug('Cached Token seems expired') + raise InvalidUserToken('Token authorization failed') def _cache_store(self, token_id, data): """Store value into memcache. From 46f2cc89126cdf23ea7e035b6c8673930d40caf2 Mon Sep 17 00:00:00 2001 From: Morgan Fainberg Date: Thu, 19 Jun 2014 15:50:41 -0700 Subject: [PATCH 120/120] Moving middleware to new location Move the middleware code to the new location within the keystonemiddleware directory. --- __init__.py => keystonemiddleware/__init__.py | 0 auth_token.py => keystonemiddleware/auth_token.py | 0 memcache_crypt.py => keystonemiddleware/memcache_crypt.py | 0 s3_token.py => keystonemiddleware/s3_token.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename __init__.py => keystonemiddleware/__init__.py (100%) rename auth_token.py => keystonemiddleware/auth_token.py (100%) rename memcache_crypt.py => keystonemiddleware/memcache_crypt.py (100%) rename s3_token.py => keystonemiddleware/s3_token.py (100%) diff --git a/__init__.py b/keystonemiddleware/__init__.py similarity index 100% rename from __init__.py rename to keystonemiddleware/__init__.py diff --git a/auth_token.py b/keystonemiddleware/auth_token.py similarity index 100% rename from auth_token.py rename to keystonemiddleware/auth_token.py diff --git a/memcache_crypt.py b/keystonemiddleware/memcache_crypt.py similarity index 100% rename from memcache_crypt.py rename to keystonemiddleware/memcache_crypt.py diff --git a/s3_token.py b/keystonemiddleware/s3_token.py similarity index 100% rename from s3_token.py rename to keystonemiddleware/s3_token.py