Current Path :
/
usr
/
lib
/
python2.6
/
site-packages
/
requestbuilder
/
Or
Select Your Path :
Upload File :
New :
File
Dir
//usr/lib/python2.6/site-packages/requestbuilder/request.py
# Copyright (c) 2012-2016 Hewlett Packard Enterprise Development LP # # Permission to use, copy, modify, and/or distribute this software for # any purpose with or without fee is hereby granted, provided that the # above copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import absolute_import, print_function import argparse import copy from functools import partial import logging import os.path import sys import textwrap import warnings import six from requestbuilder import EMPTY, PARAMS from requestbuilder.command import BaseCommand from requestbuilder.exceptions import ServerError from requestbuilder.service import BaseService from requestbuilder.util import aggregate_subclass_fields from requestbuilder.xmlparse import parse_listdelimited_aws_xml class BaseRequest(BaseCommand): """ The basis for a command line tool that represents a request. The data for this request are stored in a few instance members: - method: the HTTP method to use (e.g. 'GET'). Defaults to self.METHOD. - path: the path to append to the service's path (e.g. 'sub/dir') - headers: a dict of HTTP headers - params: a dict of query parameters - body: a string or file object containing a request body, or a dict to pass to the server as form data This specialization of BaseCommand that implements main() as a three-step process: - preprocess(): do any processing needed before sending the request, such as parsing complex command line arguments and storing them in self.params, self.headers, and so forth. - send(): send this request to the server using the data stored in its attributes, parse it using self.parse_result(), and return it - postprocess(): given a parsed response, do any processing needed before main() returns the response. Most requests need only implement preprocess(). Requests whose workflows involve sending other requests often do so in postprocess(), where the result of the request is known. Important members of this class, in addition to those inherited from BaseCommand, include: - SERVICE_CLASS: a class corresponding to the web service in use - AUTH_CLASS: a class corresponding to the authentication method to use, if any - NAME: a string representing the name of this request. This defaults to the class's name. - METHOD: the HTTP method to use by default """ SERVICE_CLASS = BaseService AUTH_CLASS = None NAME = None METHOD = 'GET' DEFAULT_ROUTES = (PARAMS,) def __init__(self, service=None, auth=None, **kwargs): # Parts of the HTTP request to be sent to the server. self.method = self.METHOD self.path = None self.headers = {} self.params = {} self.body = '' self.files = {} # HTTP response obtained from the server self.response = None self.__configured = False self.__auth = auth self.__service = service BaseCommand.__init__(self, **kwargs) @property def auth(self): if not self.__auth and self.AUTH_CLASS is not None: # pylint: disable=not-callable self.__auth = self.AUTH_CLASS(self.config, loglevel=self.log.level) # pylint: enable=not-callable return self.__auth @auth.setter def auth(self, newval): self.__auth = newval @property def service(self): if not self.__service and self.SERVICE_CLASS is not None: self.__service = self.SERVICE_CLASS(self.config, loglevel=self.log.level) return self.__service @classmethod def from_other(cls, other, **kwargs): kwargs.setdefault('service', other.service) kwargs.setdefault('auth', other.auth) kwargs.setdefault('loglevel', other.log.level) new = cls(config=other.config, **kwargs) # That already calls configure return new def collect_arg_objs(self): arg_objs = BaseCommand.collect_arg_objs(self) if self.service is not None: arg_objs.extend( aggregate_subclass_fields(self.service.__class__, 'ARGS')) if self.auth is not None: arg_objs.extend( aggregate_subclass_fields(self.auth.__class__, 'ARGS')) return arg_objs def distribute_args(self): BaseCommand.distribute_args(self) if self.service is not None: self.service.args.update(self.args) if self.auth is not None: self.auth.args.update(self.args) def configure(self): # TODO: rename this to setup BaseCommand.configure(self) if self.service is not None: self.service.configure() if self.auth is not None: self.auth.configure() self.__configured = True @property def name(self): return self.NAME or self.__class__.__name__ def send(self): if not self.__configured: self.log.warn('send() called before configure(); bugs may result') headers = dict(self.headers or {}) headers.setdefault('User-Agent', self.suite.get_user_agent()) try: self.response = self.service.send_request( method=self.method, path=self.path, headers=headers, params=self.params, data=self.body, files=self.files, auth=self.auth) return self.parse_response(self.response) except ServerError as err: self.response = err.response try: # Empty the socket buffer so it can be reused. # Hopefully error responses won't be too large for this to be # problematic. if self.response is not None: self.response.content except RuntimeError: # The content was already consumed pass return self.handle_server_error(err) def handle_server_error(self, err): self.log.debug('-- response content --\n', extra={'append': True}) self.log.debug(self.response.text, extra={'append': True}) self.log.debug('-- end of response content --') self.log.info('result: failure') raise def parse_response(self, response): return response def log_and_parse_response(self, response, parse_func, **kwargs): # We do some extra handling here to log stuff as it comes in rather # than reading it all into memory at once. self.log.debug('-- response content --\n', extra={'append': True}) # Using Response.iter_content gives us automatic decoding, but we then # have to make the generator look like a file so etree can use it. with _IteratorFileObjAdapter(self.response.iter_content(16384)) \ as content_fileobj: logged_fileobj = _ReadLoggingFileWrapper(content_fileobj, self.log, logging.DEBUG) parsed_response = parse_func(logged_fileobj, **kwargs) self.log.debug('-- end of response content --') return parsed_response def main(self): ''' The main processing method for this type of request. In this method, inheriting classes generally populate self.headers, self.params, and self.body with information gathered from self.args or elsewhere, call self.send, and return the response. ''' self.preprocess() response = self.send() self.postprocess(response) return response def preprocess(self): pass def postprocess(self, response): pass def handle_cli_exception(self, err): if isinstance(err, ServerError): msg = '{0}: {1}'.format(os.path.basename(sys.argv[0]), err.format_for_cli()) print(msg, file=sys.stderr) if self.debug: raise sys.exit(1) else: BaseCommand.handle_cli_exception(self, err) class AWSQueryRequest(BaseRequest): API_VERSION = None FILTERS = [] LIST_TAGS = [] def _populate_parser(self, parser, arg_objs): BaseRequest._populate_parser(self, parser, arg_objs) if self.FILTERS: parser.add_argument( '--filter', metavar='NAME=VALUE', action='append', dest='filters', help='restrict results to those that meet criteria', type=partial(_parse_filter, filter_objs=self.FILTERS)) parser.epilog = self.__build_filter_help() self._arg_routes['filters'] = (None,) def process_cli_args(self): BaseRequest.process_cli_args(self) if 'filters' in self.args: self.args['Filter'] = _process_filters(self.args.pop('filters')) self._arg_routes['Filter'] = (self.params,) @property def action(self): return self.name def send(self): orig_params = self.params params = copy.deepcopy(self.params) params = self.flatten_params(self.params) params['Action'] = self.action params['Version'] = self.API_VERSION or self.service.API_VERSION redacted_params = dict(params) for key in params: if key.lower().endswith('password'): # This makes it slightly more obvious that this is redacted by # the framework and not just a string. redacted_params[key] = type( 'REDACTED', (), {'__repr__': lambda self: '<redacted>'})() self.log.info('parameters: %s', redacted_params) if self.method.upper() == 'POST': self.log.debug('sending flattened parameters as form data') self.body = params self.params = {} else: self.log.debug('sending flattened parameters as query string') self.params = params try: return BaseRequest.send(self) finally: self.params = orig_params def parse_response(self, response): # Parser for list-delimited responses like EC2's response_dict = self.log_and_parse_response( response, parse_listdelimited_aws_xml, list_tags=self.LIST_TAGS) # Strip off the root element assert len(response_dict) == 1 return response_dict[list(response_dict.keys())[0]] def flatten_params(self, args, prefix=None): ''' Given a possibly-nested dict of args and an arg routing destination, transform each element in the dict that matches the corresponding arg routing table into a simple dict containing key-value pairs suitable for use as query parameters. This implementation flattens dicts and lists into the format given by AWS query APIs, which use dotted lists of dict keys and list indices to indicate nested structures. Keys with non-boolean, non-zero values that evaluate as false are ignored. To include an empty string as a parameter, pass EMPTY (the object, not the string) as its value. Examples: in: {'InstanceId': 'i-12345678', 'PublicIp': '1.2.3.4'} out: {'InstanceId': 'i-12345678', 'PublicIp': '1.2.3.4'} in: {'RegionName': ['us-east-1', 'us-west-1']} out: {'RegionName.1': 'us-east-1', 'RegionName.2': 'us-west-1'} in: {'Filter': [{'Name': 'image-id', 'Value': ['ami-12345678']}, {'Name': 'instance-type', 'Value': ['m1.small', 't1.micro']}], 'InstanceId': ['i-24680135']} out: {'Filter.1.Name': 'image-id', 'Filter.1.Value.1': 'ami-12345678', 'Filter.2.Name': 'instance-type', 'Filter.2.Value.1': 'm1.small', 'Filter.2.Value.2': 't1.micro', 'InstanceId.1': 'i-24680135'} ''' flattened = {} if args is None: pass elif isinstance(args, dict): for (key, val) in six.iteritems(args): # Prefix.Key1, Prefix.Key2, ... if prefix: prefixed_key = '{0}.{1}'.format(prefix, key) else: prefixed_key = str(key) if isinstance(val, dict) or isinstance(val, list): flattened.update(self.flatten_params(val, prefixed_key)) elif isinstance(val, bool): flattened[prefixed_key] = str(val).lower() elif isinstance(val, file): flattened[prefixed_key] = val.read() elif isinstance(val, float): flattened[prefixed_key] = str(val) elif val or val is 0: flattened[prefixed_key] = str(val) elif val is EMPTY: flattened[prefixed_key] = '' elif isinstance(args, list): for (i_item, item) in enumerate(args, 1): # Prefix.1, Prefix.2, ... if prefix: prefixed_key = '{0}.{1}'.format(prefix, i_item) else: prefixed_key = str(i_item) if isinstance(item, dict) or isinstance(item, list): flattened.update(self.flatten_params(item, prefixed_key)) elif isinstance(item, bool): flattened[prefixed_key] = str(item).lower() elif isinstance(item, file): flattened[prefixed_key] = item.read() elif item or item == 0: flattened[prefixed_key] = str(item) elif item is EMPTY: flattened[prefixed_key] = '' else: raise TypeError('non-flattenable type: ' + args.__class__.__name__) return flattened def __build_filter_help(self, force=False): ''' Return a pre-formatted help string for all of the filters defined in self.FILTERS. The result is meant to be used as command line help output. ''' # Does not have access to self.config if '-h' not in sys.argv and '--help' not in sys.argv and not force: # Performance optimization return '' helplines = ['allowed filter names:'] for filter_obj in self.FILTERS: if filter_obj.help == argparse.SUPPRESS: continue elif filter_obj.help: first, _, rest = filter_obj.help.partition('\n') if rest.startswith(' ') and not first.startswith(' '): # First line is not uniformly indented content = first + ' ' + textwrap.dedent(rest) else: content = filter_obj.help if len(filter_obj.name) <= 20: # Short name; start on same line and pad two spaces firstline = ' {0:<20} '.format(filter_obj.name) wrapper = textwrap.TextWrapper( fix_sentence_endings=True, initial_indent=firstline, subsequent_indent=(' ' * 24)) else: # Long name; start on next line helplines.append(' ' + filter_obj.name) wrapper = textwrap.TextWrapper( fix_sentence_endings=True, initial_indent=(' ' * 24), subsequent_indent=(' ' * 24)) helplines.extend(wrapper.wrap(content)) else: # No help; everything goes on one line helplines.append(' ' + filter_obj.name) return '\n'.join(helplines) def _parse_filter(filter_str, filter_objs=None): ''' Given a "key=value" string given as a command line parameter, return a pair with the matching filter's dest member and the given value after converting it to the type expected by the filter. If this is impossible, an ArgumentTypeError will result instead. ''' # Find the appropriate filter object filter_objs = [obj for obj in (filter_objs or []) if obj.matches_argval(filter_str)] if not filter_objs: msg = '"{0}" matches no available filters'.format(filter_str) raise argparse.ArgumentTypeError(msg) return filter_objs[0].convert(filter_str) def _process_filters(cli_filters): ''' Change filters from the [(key, value), ...] format given at the command line to [{'Name': key, 'Value': [value, ...]}, ...] format, which flattens to the form the server expects. ''' filter_args = {} # Compile [(key, value), ...] pairs into {key: [value, ...], ...} for (key, val) in cli_filters or {}: filter_args.setdefault(key, []) filter_args[key].append(val) # Build the flattenable [{'Name': key, 'Value': [value, ...]}, ...] filters = [{'Name': name, 'Value': values} for (name, values) in six.iteritems(filter_args)] return filters class _IteratorFileObjAdapter(object): def __init__(self, source): self._source = source self._buflist = [] self._closed = False self._len = 0 def __enter__(self): return self def __exit__(self, *args): self.close() @property def closed(self): return self._closed def close(self): if not self._closed: self._buflist = None self._closed = True def read(self, size=-1): if size is None or size < 0: for chunk in self._source: self._buflist.append(chunk) result = ''.join(self._buflist) self._buflist = [] self._len = 0 else: while self._len < size: try: chunk = next(self._source) self._buflist.append(chunk) self._len += len(chunk) except StopIteration: break result = ''.join(self._buflist) extra_len = len(result) - size self._buflist = [] self._len = 0 if extra_len > 0: self._buflist = [result[-extra_len:]] self._len = extra_len result = result[:-extra_len] return result class _ReadLoggingFileWrapper(object): def __init__(self, fileobj, logger, level): self.fileobj = fileobj self.logger = logger self.level = level def read(self, size=-1): chunk = self.fileobj.read(size) self.logger.log(self.level, chunk, extra={'append': True}) return chunk
