Package googleapiclient :: Module discovery |
[hide private]
[frames] | no frames]

# Source Code for Module googleapiclient.discovery

       1  # Copyright 2014 Google Inc. All Rights Reserved.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Client for discovery based APIs.
      16
      17  A client library for Google's discovery based APIs.
      18  """
      19  from __future__ import absolute_import
      20  import six
      21  from six.moves import zip
      22
      23  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      24  __all__ = [
      25      'build',
      26      'build_from_document',
      27      'fix_method_name',
      28      'key2param',
      29      ]
      30
      31  from six import BytesIO
      32  from six.moves import http_client
      33  from six.moves.urllib.parse import urlencode, urlparse, urljoin, \
      34    urlunparse, parse_qsl
      35
      36  # Standard library imports
      37  import copy
      38  try:
      39    from email.generator import BytesGenerator
      40  except ImportError:
      41    from email.generator import Generator as BytesGenerator
      42  from email.mime.multipart import MIMEMultipart
      43  from email.mime.nonmultipart import MIMENonMultipart
      44  import json
      45  import keyword
      46  import logging
      47  import mimetypes
      48  import os
      49  import re
      50
      51  # Third-party imports
      52  import httplib2
      53  import uritemplate
      54
      55  # Local imports
      56  from googleapiclient import mimeparse
      57  from googleapiclient.errors import HttpError
      58  from googleapiclient.errors import InvalidJsonError
      59  from googleapiclient.errors import MediaUploadSizeError
      60  from googleapiclient.errors import UnacceptableMimeTypeError
      61  from googleapiclient.errors import UnknownApiNameOrVersion
      62  from googleapiclient.errors import UnknownFileType
      63  from googleapiclient.http import BatchHttpRequest
      64  from googleapiclient.http import HttpRequest
      65  from googleapiclient.http import MediaFileUpload
      66  from googleapiclient.http import MediaUpload
      67  from googleapiclient.model import JsonModel
      68  from googleapiclient.model import MediaModel
      69  from googleapiclient.model import RawModel
      70  from googleapiclient.schema import Schemas
      71  from oauth2client.client import GoogleCredentials
      72  from oauth2client.util import _add_query_parameter
      73  from oauth2client.util import positional
      74
      75
      76  # The client library requires a version of httplib2 that supports RETRIES.
      77  httplib2.RETRIES = 1
      78
      79  logger = logging.getLogger(__name__)
      80
      81  URITEMPLATE = re.compile('{[^}]*}')
      82  VARNAME = re.compile('[a-zA-Z0-9_-]+')
      83  DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
      84                   '{api}/{apiVersion}/rest')
      85  V1_DISCOVERY_URI = DISCOVERY_URI
      86  V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?'
      87                      'version={apiVersion}')
      88  DEFAULT_METHOD_DOC = 'A description of how to use this function'
      89  HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH'])
      90  _MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40}
      91  BODY_PARAMETER_DEFAULT_VALUE = {
      92      'description': 'The request body.',
      93      'type': 'object',
      94      'required': True,
      95  }
      96  MEDIA_BODY_PARAMETER_DEFAULT_VALUE = {
      97      'description': ('The filename of the media request body, or an instance '
      98                      'of a MediaUpload object.'),
      99      'type': 'string',
     100      'required': False,
     101  }
     102
     103  # Parameters accepted by the stack, but not visible via discovery.
     104  # TODO(dhermes): Remove 'userip' in 'v2'.
     105  STACK_QUERY_PARAMETERS = frozenset(['trace', 'pp', 'userip', 'strict'])
     106  STACK_QUERY_PARAMETER_DEFAULT_VALUE = {'type': 'string', 'location': 'query'}
     107
     108  # Library-specific reserved words beyond Python keywords.
     109  RESERVED_WORDS = frozenset(['body'])
     110
     111  # patch _write_lines to avoid munging '\r' into '\n'
     112  # ( https://bugs.python.org/issue18886 https://bugs.python.org/issue19003 )
     113 -class _BytesGenerator(BytesGenerator):
     114    _write_lines = BytesGenerator.write
     115
     116 -def fix_method_name(name):
     117    """Fix method names to avoid reserved word conflicts.
     118
     119    Args:
     120      name: string, method name.
     121
     122    Returns:
     123      The name with a '_' prefixed if the name is a reserved word.
     124    """
     125    if keyword.iskeyword(name) or name in RESERVED_WORDS:
     126      return name + '_'
     127    else:
     128      return name
     129
     130
     131 -def key2param(key):
     132    """Converts key names into parameter names.
     133
     134    For example, converting "max-results" -> "max_results"
     135
     136    Args:
     137      key: string, the method key name.
     138
     139    Returns:
     140      A safe method name based on the key name.
     141    """
     142    result = []
     143    key = list(key)
     144    if not key[0].isalpha():
     145      result.append('x')
     146    for c in key:
     147      if c.isalnum():
     148        result.append(c)
     149      else:
     150        result.append('_')
     151
     152    return ''.join(result)
     153
     154
     155  @positional(2)
     156 -def build(serviceName,
     157            version,
     158            http=None,
     159            discoveryServiceUrl=DISCOVERY_URI,
     160            developerKey=None,
     161            model=None,
     162            requestBuilder=HttpRequest,
     163            credentials=None,
     164            cache_discovery=True,
     165            cache=None):
     166    """Construct a Resource for interacting with an API.
     167
     168    Construct a Resource object for interacting with an API. The serviceName and
     169    version are the names from the Discovery service.
     170
     171    Args:
     172      serviceName: string, name of the service.
     173      version: string, the version of the service.
     174      http: httplib2.Http, An instance of httplib2.Http or something that acts
     175        like it that HTTP requests will be made through.
     176      discoveryServiceUrl: string, a URI Template that points to the location of
     177        the discovery service. It should have two parameters {api} and
     178        {apiVersion} that when filled in produce an absolute URI to the discovery
     179        document for that service.
     180      developerKey: string, key obtained from
     181        https://code.google.com/apis/console.
     182      model: googleapiclient.Model, converts to and from the wire format.
     183      requestBuilder: googleapiclient.http.HttpRequest, encapsulator for an HTTP
     184        request.
     185      credentials: oauth2client.Credentials, credentials to be used for
     186        authentication.
     187      cache_discovery: Boolean, whether or not to cache the discovery doc.
     188      cache: googleapiclient.discovery_cache.base.CacheBase, an optional
     189        cache object for the discovery documents.
     190
     191    Returns:
     192      A Resource object with methods for interacting with the service.
     193    """
     194    params = {
     195        'api': serviceName,
     196        'apiVersion': version
     197        }
     198
     199    if http is None:
     200      http = httplib2.Http()
     201
     202    for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,):
     203      requested_url = uritemplate.expand(discovery_url, params)
     204
     205      try:
     206        content = _retrieve_discovery_doc(requested_url, http, cache_discovery,
     207                                          cache)
     208        return build_from_document(content, base=discovery_url, http=http,
     209            developerKey=developerKey, model=model, requestBuilder=requestBuilder,
     210            credentials=credentials)
     211      except HttpError as e:
     212        if e.resp.status == http_client.NOT_FOUND:
     213          continue
     214        else:
     215          raise e
     216
     217    raise UnknownApiNameOrVersion(
     218          "name: %s  version: %s" % (serviceName, version))
     219
     220
     221 -def _retrieve_discovery_doc(url, http, cache_discovery, cache=None):
     222    """Retrieves the discovery_doc from cache or the internet.
     223
     224    Args:
     225      url: string, the URL of the discovery document.
     226      http: httplib2.Http, An instance of httplib2.Http or something that acts
     227        like it through which HTTP requests will be made.
     228      cache_discovery: Boolean, whether or not to cache the discovery doc.
     229      cache: googleapiclient.discovery_cache.base.Cache, an optional cache
     230        object for the discovery documents.
     231
     232    Returns:
     233      A unicode string representation of the discovery document.
     234    """
     235    if cache_discovery:
     236      from . import discovery_cache
     237      from .discovery_cache import base
     238      if cache is None:
     239        cache = discovery_cache.autodetect()
     240      if cache:
     241        content = cache.get(url)
     242        if content:
     243          return content
     244
     245    actual_url = url
     246    # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
     247    # variable that contains the network address of the client sending the
     248    # request. If it exists then add that to the request for the discovery
     249    # document to avoid exceeding the quota on discovery requests.
     250    if 'REMOTE_ADDR' in os.environ:
     251      actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR'])
     252    logger.info('URL being requested: GET %s', actual_url)
     253
     254    resp, content = http.request(actual_url)
     255
     256    if resp.status >= 400:
     257      raise HttpError(resp, content, uri=actual_url)
     258
     259    try:
     260      content = content.decode('utf-8')
     261    except AttributeError:
     262      pass
     263
     264    try:
     265      service = json.loads(content)
     266    except ValueError as e:
     267      logger.error('Failed to parse as JSON: ' + content)
     268      raise InvalidJsonError()
     269    if cache_discovery and cache:
     270      cache.set(url, content)
     271    return content
     272
     273
     274  @positional(1)
     275 -def build_from_document(
     276      service,
     277      base=None,
     278      future=None,
     279      http=None,
     280      developerKey=None,
     281      model=None,
     282      requestBuilder=HttpRequest,
     283      credentials=None):
     284    """Create a Resource for interacting with an API.
     285
     286    Same as `build()`, but constructs the Resource object from a discovery
     287    document that is it given, as opposed to retrieving one over HTTP.
     288
     289    Args:
     290      service: string or object, the JSON discovery document describing the API.
     291        The value passed in may either be the JSON string or the deserialized
     292        JSON.
     293      base: string, base URI for all HTTP requests, usually the discovery URI.
     294        This parameter is no longer used as rootUrl and servicePath are included
     295        within the discovery document. (deprecated)
     296      future: string, discovery document with future capabilities (deprecated).
     297      http: httplib2.Http, An instance of httplib2.Http or something that acts
     298        like it that HTTP requests will be made through.
     299      developerKey: string, Key for controlling API usage, generated
     300        from the API Console.
     301      model: Model class instance that serializes and de-serializes requests and
     302        responses.
     303      requestBuilder: Takes an http request and packages it up to be executed.
     304      credentials: object, credentials to be used for authentication.
     305
     306    Returns:
     307      A Resource object with methods for interacting with the service.
     308    """
     309
     310    if http is None:
     311      http = httplib2.Http()
     312
     313    # future is no longer used.
     314    future = {}
     315
     316    if isinstance(service, six.string_types):
     317      service = json.loads(service)
     318    base = urljoin(service['rootUrl'], service['servicePath'])
     319    schema = Schemas(service)
     320
     321    if credentials:
     322      # If credentials were passed in, we could have two cases:
     323      # 1. the scopes were specified, in which case the given credentials
     324      #    are used for authorizing the http;
     325      # 2. the scopes were not provided (meaning the Application Default
     326      #    Credentials are to be used). In this case, the Application Default
     327      #    Credentials are built and used instead of the original credentials.
     328      #    If there are no scopes found (meaning the given service requires no
     329      #    authentication), there is no authorization of the http.
     330      if (isinstance(credentials, GoogleCredentials) and
     331          credentials.create_scoped_required()):
     332        scopes = service.get('auth', {}).get('oauth2', {}).get('scopes', {})
     333        if scopes:
     334          credentials = credentials.create_scoped(list(scopes.keys()))
     335        else:
     336          # No need to authorize the http object
     337          # if the service does not require authentication.
     338          credentials = None
     339
     340      if credentials:
     341        http = credentials.authorize(http)
     342
     343    if model is None:
     344      features = service.get('features', [])
     345      model = JsonModel('dataWrapper' in features)
     346    return Resource(http=http, baseUrl=base, model=model,
     347                    developerKey=developerKey, requestBuilder=requestBuilder,
     348                    resourceDesc=service, rootDesc=service, schema=schema)
     349
     350
     351 -def _cast(value, schema_type):
     352    """Convert value to a string based on JSON Schema type.
     353
     354    See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
     355    JSON Schema.
     356
     357    Args:
     358      value: any, the value to convert
     359      schema_type: string, the type that value should be interpreted as
     360
     361    Returns:
     362      A string representation of 'value' based on the schema_type.
     363    """
     364    if schema_type == 'string':
     365      if type(value) == type('') or type(value) == type(u''):
     366        return value
     367      else:
     368        return str(value)
     369    elif schema_type == 'integer':
     370      return str(int(value))
     371    elif schema_type == 'number':
     372      return str(float(value))
     373    elif schema_type == 'boolean':
     374      return str(bool(value)).lower()
     375    else:
     376      if type(value) == type('') or type(value) == type(u''):
     377        return value
     378      else:
     379        return str(value)
     380
     381
     382 -def _media_size_to_long(maxSize):
     383    """Convert a string media size, such as 10GB or 3TB into an integer.
     384
     385    Args:
     386      maxSize: string, size as a string, such as 2MB or 7GB.
     387
     388    Returns:
     389      The size as an integer value.
     390    """
     391    if len(maxSize) < 2:
     392      return 0
     393    units = maxSize[-2:].upper()
     394    bit_shift = _MEDIA_SIZE_BIT_SHIFTS.get(units)
     395    if bit_shift is not None:
     396      return int(maxSize[:-2]) << bit_shift
     397    else:
     398      return int(maxSize)
     399
     400
     401 -def _media_path_url_from_info(root_desc, path_url):
     402    """Creates an absolute media path URL.
     403
     404    Constructed using the API root URI and service path from the discovery
     405    document and the relative path for the API method.
     406
     407    Args:
     408      root_desc: Dictionary; the entire original deserialized discovery document.
     409      path_url: String; the relative URL for the API method. Relative to the API
     410          root, which is specified in the discovery document.
     411
     412    Returns:
     413      String; the absolute URI for media upload for the API method.
     414    """
     415    return '%(root)supload/%(service_path)s%(path)s' % {
     416        'root': root_desc['rootUrl'],
     417        'service_path': root_desc['servicePath'],
     418        'path': path_url,
     419    }
     420
     421
     422 -def _fix_up_parameters(method_desc, root_desc, http_method):
     423    """Updates parameters of an API method with values specific to this library.
     424
     425    Specifically, adds whatever global parameters are specified by the API to the
     426    parameters for the individual method. Also adds parameters which don't
     427    appear in the discovery document, but are available to all discovery based
     428    APIs (these are listed in STACK_QUERY_PARAMETERS).
     429
     430    SIDE EFFECTS: This updates the parameters dictionary object in the method
     431    description.
     432
     433    Args:
     434      method_desc: Dictionary with metadata describing an API method. Value comes
     435          from the dictionary of methods stored in the 'methods' key in the
     436          deserialized discovery document.
     437      root_desc: Dictionary; the entire original deserialized discovery document.
     438      http_method: String; the HTTP method used to call the API method described
     439          in method_desc.
     440
     441    Returns:
     442      The updated Dictionary stored in the 'parameters' key of the method
     443          description dictionary.
     444    """
     445    parameters = method_desc.setdefault('parameters', {})
     446
     447    # Add in the parameters common to all methods.
     448    for name, description in six.iteritems(root_desc.get('parameters', {})):
     449      parameters[name] = description
     450
     451    # Add in undocumented query parameters.
     452    for name in STACK_QUERY_PARAMETERS:
     453      parameters[name] = STACK_QUERY_PARAMETER_DEFAULT_VALUE.copy()
     454
     455    # Add 'body' (our own reserved word) to parameters if the method supports
     456    # a request payload.
     457    if http_method in HTTP_PAYLOAD_METHODS and 'request' in method_desc:
     458      body = BODY_PARAMETER_DEFAULT_VALUE.copy()
     459      body.update(method_desc['request'])
     460      parameters['body'] = body
     461
     462    return parameters
     463
     464
     465 -def _fix_up_media_upload(method_desc, root_desc, path_url, parameters):
     466    """Updates parameters of API by adding 'media_body' if supported by method.
     467
     468    SIDE EFFECTS: If the method supports media upload and has a required body,
     469    sets body to be optional (required=False) instead. Also, if there is a
     470    'mediaUpload' in the method description, adds 'media_upload' key to
     471    parameters.
     472
     473    Args:
     474      method_desc: Dictionary with metadata describing an API method. Value comes
     475          from the dictionary of methods stored in the 'methods' key in the
     476          deserialized discovery document.
     477      root_desc: Dictionary; the entire original deserialized discovery document.
     478      path_url: String; the relative URL for the API method. Relative to the API
     479          root, which is specified in the discovery document.
     480      parameters: A dictionary describing method parameters for method described
     481          in method_desc.
     482
     483    Returns:
     484      Triple (accept, max_size, media_path_url) where:
     485        - accept is a list of strings representing what content types are
     486          accepted for media upload. Defaults to empty list if not in the
     487          discovery document.
     488        - max_size is a long representing the max size in bytes allowed for a
     489          media upload. Defaults to 0L if not in the discovery document.
     490        - media_path_url is a String; the absolute URI for media upload for the
     491          API method. Constructed using the API root URI and service path from
     492          the discovery document and the relative path for the API method. If
     493          media upload is not supported, this is None.
     494    """
     495    media_upload = method_desc.get('mediaUpload', {})
     496    accept = media_upload.get('accept', [])
     497    max_size = _media_size_to_long(media_upload.get('maxSize', ''))
     498    media_path_url = None
     499
     500    if media_upload:
     501      media_path_url = _media_path_url_from_info(root_desc, path_url)
     502      parameters['media_body'] = MEDIA_BODY_PARAMETER_DEFAULT_VALUE.copy()
     503      if 'body' in parameters:
     504        parameters['body']['required'] = False
     505
     506    return accept, max_size, media_path_url
     507
     508
     509 -def _fix_up_method_description(method_desc, root_desc):
     510    """Updates a method description in a discovery document.
     511
     512    SIDE EFFECTS: Changes the parameters dictionary in the method description with
     513    extra parameters which are used locally.
     514
     515    Args:
     516      method_desc: Dictionary with metadata describing an API method. Value comes
     517          from the dictionary of methods stored in the 'methods' key in the
     518          deserialized discovery document.
     519      root_desc: Dictionary; the entire original deserialized discovery document.
     520
     521    Returns:
     522      Tuple (path_url, http_method, method_id, accept, max_size, media_path_url)
     523      where:
     524        - path_url is a String; the relative URL for the API method. Relative to
     525          the API root, which is specified in the discovery document.
     526        - http_method is a String; the HTTP method used to call the API method
     527          described in the method description.
     528        - method_id is a String; the name of the RPC method associated with the
     529          API method, and is in the method description in the 'id' key.
     530        - accept is a list of strings representing what content types are
     531          accepted for media upload. Defaults to empty list if not in the
     532          discovery document.
     533        - max_size is a long representing the max size in bytes allowed for a
     534          media upload. Defaults to 0L if not in the discovery document.
     535        - media_path_url is a String; the absolute URI for media upload for the
     536          API method. Constructed using the API root URI and service path from
     537          the discovery document and the relative path for the API method. If
     538          media upload is not supported, this is None.
     539    """
     540    path_url = method_desc['path']
     541    http_method = method_desc['httpMethod']
     542    method_id = method_desc['id']
     543
     544    parameters = _fix_up_parameters(method_desc, root_desc, http_method)
     545    # Order is important. `_fix_up_media_upload` needs `method_desc` to have a
     546    # 'parameters' key and needs to know if there is a 'body' parameter because it
     547    # also sets a 'media_body' parameter.
     548    accept, max_size, media_path_url = _fix_up_media_upload(
     549        method_desc, root_desc, path_url, parameters)
     550
     551    return path_url, http_method, method_id, accept, max_size, media_path_url
     552
     553
     554 -def _urljoin(base, url):
     555    """Custom urljoin replacement supporting : before / in url."""
     556    # In general, it's unsafe to simply join base and url. However, for
     557    # the case of discovery documents, we know:
     558    #  * base will never contain params, query, or fragment
     559    #  * url will never contain a scheme or net_loc.
     560    # In general, this means we can safely join on /; we just need to
     561    # ensure we end up with precisely one / joining base and url. The
     562    # exception here is the case of media uploads, where url will be an
     563    # absolute url.
     564    if url.startswith('http://') or url.startswith('https://'):
     565      return urljoin(base, url)
     566    new_base = base if base.endswith('/') else base + '/'
     567    new_url = url[1:] if url.startswith('/') else url
     568    return new_base + new_url
     569
     570
     571  # TODO(dhermes): Convert this class to ResourceMethod and make it callable
     572 -class ResourceMethodParameters(object):
     573    """Represents the parameters associated with a method.
     574
     575    Attributes:
     576      argmap: Map from method parameter name (string) to query parameter name
     577          (string).
     578      required_params: List of required parameters (represented by parameter
     579          name as string).
     580      repeated_params: List of repeated parameters (represented by parameter
     581          name as string).
     582      pattern_params: Map from method parameter name (string) to regular
     583          expression (as a string). If the pattern is set for a parameter, the
     584          value for that parameter must match the regular expression.
     585      query_params: List of parameters (represented by parameter name as string)
     586          that will be used in the query string.
     587      path_params: Set of parameters (represented by parameter name as string)
     588          that will be used in the base URL path.
     589      param_types: Map from method parameter name (string) to parameter type. Type
     590          can be any valid JSON schema type; valid values are 'any', 'array',
     591          'boolean', 'integer', 'number', 'object', or 'string'. Reference:
     592          http://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1
     593      enum_params: Map from method parameter name (string) to list of strings,
     594         where each list of strings is the list of acceptable enum values.
     595    """
     596
     597 -  def __init__(self, method_desc):
     598      """Constructor for ResourceMethodParameters.
     599
     600      Sets default values and defers to set_parameters to populate.
     601
     602      Args:
     603        method_desc: Dictionary with metadata describing an API method. Value
     604            comes from the dictionary of methods stored in the 'methods' key in
     605            the deserialized discovery document.
     606      """
     607      self.argmap = {}
     608      self.required_params = []
     609      self.repeated_params = []
     610      self.pattern_params = {}
     611      self.query_params = []
     612      # TODO(dhermes): Change path_params to a list if the extra URITEMPLATE
     613      #                parsing is gotten rid of.
     614      self.path_params = set()
     615      self.param_types = {}
     616      self.enum_params = {}
     617
     618      self.set_parameters(method_desc)
     619
     620 -  def set_parameters(self, method_desc):
     621      """Populates maps and lists based on method description.
     622
     623      Iterates through each parameter for the method and parses the values from
     624      the parameter dictionary.
     625
     626      Args:
     627        method_desc: Dictionary with metadata describing an API method. Value
     628            comes from the dictionary of methods stored in the 'methods' key in
     629            the deserialized discovery document.
     630      """
     631      for arg, desc in six.iteritems(method_desc.get('parameters', {})):
     632        param = key2param(arg)
     633        self.argmap[param] = arg
     634
     635        if desc.get('pattern'):
     636          self.pattern_params[param] = desc['pattern']
     637        if desc.get('enum'):
     638          self.enum_params[param] = desc['enum']
     639        if desc.get('required'):
     640          self.required_params.append(param)
     641        if desc.get('repeated'):
     642          self.repeated_params.append(param)
     643        if desc.get('location') == 'query':
     644          self.query_params.append(param)
     645        if desc.get('location') == 'path':
     646          self.path_params.add(param)
     647        self.param_types[param] = desc.get('type', 'string')
     648
     649      # TODO(dhermes): Determine if this is still necessary. Discovery based APIs
     650      #                should have all path parameters already marked with
     651      #                'location: path'.
     652      for match in URITEMPLATE.finditer(method_desc['path']):
     653        for namematch in VARNAME.finditer(match.group(0)):
     654          name = key2param(namematch.group(0))
     655          self.path_params.add(name)
     656          if name in self.query_params:
     657            self.query_params.remove(name)
     658
     659
     660 -def createMethod(methodName, methodDesc, rootDesc, schema):
     661    """Creates a method for attaching to a Resource.
     662
     663    Args:
     664      methodName: string, name of the method to use.
     665      methodDesc: object, fragment of deserialized discovery document that
     666        describes the method.
     667      rootDesc: object, the entire deserialized discovery document.
     668      schema: object, mapping of schema names to schema descriptions.
     669    """
     670    methodName = fix_method_name(methodName)
     671    (pathUrl, httpMethod, methodId, accept,
     672     maxSize, mediaPathUrl) = _fix_up_method_description(methodDesc, rootDesc)
     673
     674    parameters = ResourceMethodParameters(methodDesc)
     675
     676    def method(self, **kwargs):
     677      # Don't bother with doc string, it will be over-written by createMethod.
     678
     679      for name in six.iterkeys(kwargs):
     680        if name not in parameters.argmap:
     681          raise TypeError('Got an unexpected keyword argument "%s"' % name)
     682
     683      # Remove args that have a value of None.
     684      keys = list(kwargs.keys())
     685      for name in keys:
     686        if kwargs[name] is None:
     687          del kwargs[name]
     688
     689      for name in parameters.required_params:
     690        if name not in kwargs:
     691          raise TypeError('Missing required parameter "%s"' % name)
     692
     693      for name, regex in six.iteritems(parameters.pattern_params):
     694        if name in kwargs:
     695          if isinstance(kwargs[name], six.string_types):
     696            pvalues = [kwargs[name]]
     697          else:
     698            pvalues = kwargs[name]
     699          for pvalue in pvalues:
     700            if re.match(regex, pvalue) is None:
     701              raise TypeError(
     702                  'Parameter "%s" value "%s" does not match the pattern "%s"' %
     703                  (name, pvalue, regex))
     704
     705      for name, enums in six.iteritems(parameters.enum_params):
     706        if name in kwargs:
     707          # We need to handle the case of a repeated enum
     708          # name differently, since we want to handle both
     709          # arg='value' and arg=['value1', 'value2']
     710          if (name in parameters.repeated_params and
     711              not isinstance(kwargs[name], six.string_types)):
     712            values = kwargs[name]
     713          else:
     714            values = [kwargs[name]]
     715          for value in values:
     716            if value not in enums:
     717              raise TypeError(
     718                  'Parameter "%s" value "%s" is not an allowed value in "%s"' %
     719                  (name, value, str(enums)))
     720
     721      actual_query_params = {}
     722      actual_path_params = {}
     723      for key, value in six.iteritems(kwargs):
     724        to_type = parameters.param_types.get(key, 'string')
     725        # For repeated parameters we cast each member of the list.
     726        if key in parameters.repeated_params and type(value) == type([]):
     727          cast_value = [_cast(x, to_type) for x in value]
     728        else:
     729          cast_value = _cast(value, to_type)
     730        if key in parameters.query_params:
     731          actual_query_params[parameters.argmap[key]] = cast_value
     732        if key in parameters.path_params:
     733          actual_path_params[parameters.argmap[key]] = cast_value
     734      body_value = kwargs.get('body', None)
     735      media_filename = kwargs.get('media_body', None)
     736
     737      if self._developerKey:
     738        actual_query_params['key'] = self._developerKey
     739
     740      model = self._model
     741      if methodName.endswith('_media'):
     742        model = MediaModel()
     743      elif 'response' not in methodDesc:
     744        model = RawModel()
     745
     746      headers = {}
     747      headers, params, query, body = model.request(headers,
     748          actual_path_params, actual_query_params, body_value)
     749
     750      expanded_url = uritemplate.expand(pathUrl, params)
     751      url = _urljoin(self._baseUrl, expanded_url + query)
     752
     753      resumable = None
     754      multipart_boundary = ''
     755
     756      if media_filename:
     757        # Ensure we end up with a valid MediaUpload object.
     758        if isinstance(media_filename, six.string_types):
     759          (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
     760          if media_mime_type is None:
     761            raise UnknownFileType(media_filename)
     762          if not mimeparse.best_match([media_mime_type], ','.join(accept)):
     763            raise UnacceptableMimeTypeError(media_mime_type)
     764          media_upload = MediaFileUpload(media_filename,
     765                                         mimetype=media_mime_type)
     766        elif isinstance(media_filename, MediaUpload):
     767          media_upload = media_filename
     768        else:
     769          raise TypeError('media_filename must be str or MediaUpload.')
     770
     771        # Check the maxSize
     772        if media_upload.size() is not None and media_upload.size() > maxSize > 0:
     773          raise MediaUploadSizeError("Media larger than: %s" % maxSize)
     774
     775        # Use the media path uri for media uploads
     776        expanded_url = uritemplate.expand(mediaPathUrl, params)
     777        url = _urljoin(self._baseUrl, expanded_url + query)
     778        if media_upload.resumable():
     779          url = _add_query_parameter(url, 'uploadType', 'resumable')
     780
     781        if media_upload.resumable():
     782          # This is all we need to do for resumable, if the body exists it gets
     783          # sent in the first request, otherwise an empty body is sent.
     784          resumable = media_upload
     785        else:
     786          # A non-resumable upload
     787          if body is None:
     788            # This is a simple media upload
     789            headers['content-type'] = media_upload.mimetype()
     790            body = media_upload.getbytes(0, media_upload.size())
     791            url = _add_query_parameter(url, 'uploadType', 'media')
     792          else:
     793            # This is a multipart/related upload.
     794            msgRoot = MIMEMultipart('related')
     795            # msgRoot should not write out it's own headers
     796            setattr(msgRoot, '_write_headers', lambda self: None)
     797
     798            # attach the body as one part
     799            msg = MIMENonMultipart(*headers['content-type'].split('/'))
     800            msg.set_payload(body)
     801            msgRoot.attach(msg)
     802
     803            # attach the media as the second part
     804            msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
     805            msg['Content-Transfer-Encoding'] = 'binary'
     806
     807            payload = media_upload.getbytes(0, media_upload.size())
     808            msg.set_payload(payload)
     809            msgRoot.attach(msg)
     810            # encode the body: note that we can't use `as_string`, because
     811            # it plays games with `From ` lines.
     812            fp = BytesIO()
     813            g = _BytesGenerator(fp, mangle_from_=False)
     814            g.flatten(msgRoot, unixfrom=False)
     815            body = fp.getvalue()
     816
     817            multipart_boundary = msgRoot.get_boundary()
     818            headers['content-type'] = ('multipart/related; '
     819                                       'boundary="%s"') % multipart_boundary
     820            url = _add_query_parameter(url, 'uploadType', 'multipart')
     821
     822      logger.info('URL being requested: %s %s' % (httpMethod,url))
     823      return self._requestBuilder(self._http,
     824                                  model.response,
     825                                  url,
     826                                  method=httpMethod,
     827                                  body=body,
     828                                  headers=headers,
     829                                  methodId=methodId,
     830                                  resumable=resumable)
     831
     832    docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
     833    if len(parameters.argmap) > 0:
     834      docs.append('Args:\n')
     835
     836    # Skip undocumented params and params common to all methods.
     837    skip_parameters = list(rootDesc.get('parameters', {}).keys())
     838    skip_parameters.extend(STACK_QUERY_PARAMETERS)
     839
     840    all_args = list(parameters.argmap.keys())
     841    args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
     842
     843    # Move body to the front of the line.
     844    if 'body' in all_args:
     845      args_ordered.append('body')
     846
     847    for name in all_args:
     848      if name not in args_ordered:
     849        args_ordered.append(name)
     850
     851    for arg in args_ordered:
     852      if arg in skip_parameters:
     853        continue
     854
     855      repeated = ''
     856      if arg in parameters.repeated_params:
     857        repeated = ' (repeated)'
     858      required = ''
     859      if arg in parameters.required_params:
     860        required = ' (required)'
     861      paramdesc = methodDesc['parameters'][parameters.argmap[arg]]
     862      paramdoc = paramdesc.get('description', 'A parameter')
     863      if '$ref' in paramdesc:
     864        docs.append(
     865            ('  %s: object, %s%s%s\n    The object takes the'
     866            ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
     867              schema.prettyPrintByName(paramdesc['$ref'])))
     868      else:
     869        paramtype = paramdesc.get('type', 'string')
     870        docs.append('  %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
     871                                            repeated))
     872      enum = paramdesc.get('enum', [])
     873      enumDesc = paramdesc.get('enumDescriptions', [])
     874      if enum and enumDesc:
     875        docs.append('    Allowed values\n')
     876        for (name, desc) in zip(enum, enumDesc):
     877          docs.append('      %s - %s\n' % (name, desc))
     878    if 'response' in methodDesc:
     879      if methodName.endswith('_media'):
     880        docs.append('\nReturns:\n  The media object as a string.\n\n    ')
     881      else:
     882        docs.append('\nReturns:\n  An object of the form:\n\n    ')
     883        docs.append(schema.prettyPrintSchema(methodDesc['response']))
     884
     885    setattr(method, '__doc__', ''.join(docs))
     886    return (methodName, method)
     887
     888
     889 -def createNextMethod(methodName):
     890    """Creates any _next methods for attaching to a Resource.
     891
     892    The _next methods allow for easy iteration through list() responses.
     893
     894    Args:
     895      methodName: string, name of the method to use.
     896    """
     897    methodName = fix_method_name(methodName)
     898
     899    def methodNext(self, previous_request, previous_response):
     900      """Retrieves the next page of results.
     901
     902  Args:
     903    previous_request: The request for the previous page. (required)
     904    previous_response: The response from the request for the previous page. (required)
     905
     906  Returns:
     907    A request object that you can call 'execute()' on to request the next
     908    page. Returns None if there are no more items in the collection.
     909      """
     910      # Retrieve nextPageToken from previous_response
     911      # Use as pageToken in previous_request to create new request.
     912
     913      if 'nextPageToken' not in previous_response or not previous_response['nextPageToken']:
     914        return None
     915
     916      request = copy.copy(previous_request)
     917
     918      pageToken = previous_response['nextPageToken']
     919      parsed = list(urlparse(request.uri))
     920      q = parse_qsl(parsed[4])
     921
     922      # Find and remove old 'pageToken' value from URI
     923      newq = [(key, value) for (key, value) in q if key != 'pageToken']
     924      newq.append(('pageToken', pageToken))
     925      parsed[4] = urlencode(newq)
     926      uri = urlunparse(parsed)
     927
     928      request.uri = uri
     929
     930      logger.info('URL being requested: %s %s' % (methodName,uri))
     931
     932      return request
     933
     934    return (methodName, methodNext)
     935
     936
     937 -class Resource(object):
     938    """A class for interacting with a resource."""
     939
     940 -  def __init__(self, http, baseUrl, model, requestBuilder, developerKey,
     941                 resourceDesc, rootDesc, schema):
     942      """Build a Resource from the API description.
     943
     944      Args:
     945        http: httplib2.Http, Object to make http requests with.
     946        baseUrl: string, base URL for the API. All requests are relative to this
     947            URI.
     948        model: googleapiclient.Model, converts to and from the wire format.
     949        requestBuilder: class or callable that instantiates an
     950            googleapiclient.HttpRequest object.
     951        developerKey: string, key obtained from
     952            https://code.google.com/apis/console
     953        resourceDesc: object, section of deserialized discovery document that
     954            describes a resource. Note that the top level discovery document
     955            is considered a resource.
     956        rootDesc: object, the entire deserialized discovery document.
     957        schema: object, mapping of schema names to schema descriptions.
     958      """
     959      self._dynamic_attrs = []
     960
     961      self._http = http
     962      self._baseUrl = baseUrl
     963      self._model = model
     964      self._developerKey = developerKey
     965      self._requestBuilder = requestBuilder
     966      self._resourceDesc = resourceDesc
     967      self._rootDesc = rootDesc
     968      self._schema = schema
     969
     970      self._set_service_methods()
     971
     972 -  def _set_dynamic_attr(self, attr_name, value):
     973      """Sets an instance attribute and tracks it in a list of dynamic attributes.
     974
     975      Args:
     976        attr_name: string; The name of the attribute to be set
     977        value: The value being set on the object and tracked in the dynamic cache.
     978      """
     979      self._dynamic_attrs.append(attr_name)
     980      self.__dict__[attr_name] = value
     981
     982 -  def __getstate__(self):
     983      """Trim the state down to something that can be pickled.
     984
     985      Uses the fact that the instance variable _dynamic_attrs holds attrs that
     986      will be wiped and restored on pickle serialization.
     987      """
     988      state_dict = copy.copy(self.__dict__)
     989      for dynamic_attr in self._dynamic_attrs:
     990        del state_dict[dynamic_attr]
     991      del state_dict['_dynamic_attrs']
     992      return state_dict
     993
     994 -  def __setstate__(self, state):
     995      """Reconstitute the state of the object from being pickled.
     996
     997      Uses the fact that the instance variable _dynamic_attrs holds attrs that
     998      will be wiped and restored on pickle serialization.
     999      """
    1000      self.__dict__.update(state)
    1001      self._dynamic_attrs = []
    1002      self._set_service_methods()
    1003
    1004 -  def _set_service_methods(self):
    1005      self._add_basic_methods(self._resourceDesc, self._rootDesc, self._schema)
    1006      self._add_nested_resources(self._resourceDesc, self._rootDesc, self._schema)
    1007      self._add_next_methods(self._resourceDesc, self._schema)
    1008
    1009 -  def _add_basic_methods(self, resourceDesc, rootDesc, schema):
    1010      # If this is the root Resource, add a new_batch_http_request() method.
    1011      if resourceDesc == rootDesc:
    1012        batch_uri = '%s%s' % (
    1013          rootDesc['rootUrl'], rootDesc.get('batchPath', 'batch'))
    1014        def new_batch_http_request(callback=None):
    1015          """Create a BatchHttpRequest object based on the discovery document.
    1016
    1017          Args:
    1018            callback: callable, A callback to be called for each response, of the
    1019              form callback(id, response, exception). The first parameter is the
    1020              request id, and the second is the deserialized response object. The
    1021              third is an apiclient.errors.HttpError exception object if an HTTP
    1022              error occurred while processing the request, or None if no error
    1023              occurred.
    1024
    1025          Returns:
    1026            A BatchHttpRequest object based on the discovery document.
    1027          """
    1028          return BatchHttpRequest(callback=callback, batch_uri=batch_uri)
    1029        self._set_dynamic_attr('new_batch_http_request', new_batch_http_request)
    1030
    1031      # Add basic methods to Resource
    1032      if 'methods' in resourceDesc:
    1033        for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
    1034          fixedMethodName, method = createMethod(
    1035              methodName, methodDesc, rootDesc, schema)
    1036          self._set_dynamic_attr(fixedMethodName,
    1037                                 method.__get__(self, self.__class__))
    1038          # Add in _media methods. The functionality of the attached method will
    1039          # change when it sees that the method name ends in _media.
    1040          if methodDesc.get('supportsMediaDownload', False):
    1041            fixedMethodName, method = createMethod(
    1042                methodName + '_media', methodDesc, rootDesc, schema)
    1043            self._set_dynamic_attr(fixedMethodName,
    1044                                   method.__get__(self, self.__class__))
    1045
    1046 -  def _add_nested_resources(self, resourceDesc, rootDesc, schema):
    1047      # Add in nested resources
    1048      if 'resources' in resourceDesc:
    1049
    1050        def createResourceMethod(methodName, methodDesc):
    1051          """Create a method on the Resource to access a nested Resource.
    1052
    1053          Args:
    1054            methodName: string, name of the method to use.
    1055            methodDesc: object, fragment of deserialized discovery document that
    1056              describes the method.
    1057          """
    1058          methodName = fix_method_name(methodName)
    1059
    1060          def methodResource(self):
    1061            return Resource(http=self._http, baseUrl=self._baseUrl,
    1062                            model=self._model, developerKey=self._developerKey,
    1063                            requestBuilder=self._requestBuilder,
    1064                            resourceDesc=methodDesc, rootDesc=rootDesc,
    1065                            schema=schema)
    1066
    1067          setattr(methodResource, '__doc__', 'A collection resource.')
    1068          setattr(methodResource, '__is_resource__', True)
    1069
    1070          return (methodName, methodResource)
    1071
    1072        for methodName, methodDesc in six.iteritems(resourceDesc['resources']):
    1073          fixedMethodName, method = createResourceMethod(methodName, methodDesc)
    1074          self._set_dynamic_attr(fixedMethodName,
    1075                                 method.__get__(self, self.__class__))
    1076
    1077 -  def _add_next_methods(self, resourceDesc, schema):
    1078      # Add _next() methods
    1079      # Look for response bodies in schema that contain nextPageToken, and methods
    1080      # that take a pageToken parameter.
    1081      if 'methods' in resourceDesc:
    1082        for methodName, methodDesc in six.iteritems(resourceDesc['methods']):
    1083          if 'response' in methodDesc:
    1084            responseSchema = methodDesc['response']
    1085            if '$ref' in responseSchema:
    1086              responseSchema = schema.get(responseSchema['$ref'])
    1087            hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
    1088                                                                     {})
    1089            hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
    1090            if hasNextPageToken and hasPageToken:
    1091              fixedMethodName, method = createNextMethod(methodName + '_next')
    1092              self._set_dynamic_attr(fixedMethodName,
    1093                                     method.__get__(self, self.__class__))
    1094