Initial commit
This commit is contained in:
Senad Uka
@@ -0,0 +1,677 @@
|
||||
# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
||||
# may not use this file except in compliance with the License. A copy of
|
||||
# the License is located at
|
||||
#
|
||||
# http://aws.amazon.com/apache2.0/
|
||||
#
|
||||
# or in the "license" file accompanying this file. This file is
|
||||
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
||||
# ANY KIND, either express or implied. See the License for the specific
|
||||
# language governing permissions and limitations under the License.
|
||||
|
||||
from itertools import tee
|
||||
|
||||
from botocore.compat import six
|
||||
|
||||
import jmespath
|
||||
import json
|
||||
import base64
|
||||
import logging
|
||||
from botocore.exceptions import PaginationError
|
||||
from botocore.compat import zip
|
||||
from botocore.utils import set_value_from_jmespath, merge_dicts
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TokenEncoder(object):
|
||||
"""Encodes dictionaries into opaque strings.
|
||||
|
||||
This for the most part json dumps + base64 encoding, but also supports
|
||||
having bytes in the dictionary in addition to the types that json can
|
||||
handle by default.
|
||||
|
||||
This is intended for use in encoding pagination tokens, which in some
|
||||
cases can be complex structures and / or contain bytes.
|
||||
"""
|
||||
|
||||
def encode(self, token):
|
||||
"""Encodes a dictionary to an opaque string.
|
||||
|
||||
:type token: dict
|
||||
:param token: A dictionary containing pagination information,
|
||||
particularly the service pagination token(s) but also other boto
|
||||
metadata.
|
||||
|
||||
:rtype: str
|
||||
:returns: An opaque string
|
||||
"""
|
||||
try:
|
||||
# Try just using json dumps first to avoid having to traverse
|
||||
# and encode the dict. In 99.9999% of cases this will work.
|
||||
json_string = json.dumps(token)
|
||||
except (TypeError, UnicodeDecodeError):
|
||||
# If normal dumping failed, go through and base64 encode all bytes.
|
||||
encoded_token, encoded_keys = self._encode(token, [])
|
||||
|
||||
# Save the list of all the encoded key paths. We can safely
|
||||
# assume that no service will ever use this key.
|
||||
encoded_token['boto_encoded_keys'] = encoded_keys
|
||||
|
||||
# Now that the bytes are all encoded, dump the json.
|
||||
json_string = json.dumps(encoded_token)
|
||||
|
||||
# base64 encode the json string to produce an opaque token string.
|
||||
return base64.b64encode(json_string.encode('utf-8')).decode('utf-8')
|
||||
|
||||
def _encode(self, data, path):
|
||||
"""Encode bytes in given data, keeping track of the path traversed."""
|
||||
if isinstance(data, dict):
|
||||
return self._encode_dict(data, path)
|
||||
elif isinstance(data, list):
|
||||
return self._encode_list(data, path)
|
||||
elif isinstance(data, six.binary_type):
|
||||
return self._encode_bytes(data, path)
|
||||
else:
|
||||
return data, []
|
||||
|
||||
def _encode_list(self, data, path):
|
||||
"""Encode any bytes in a list, noting the index of what is encoded."""
|
||||
new_data = []
|
||||
encoded = []
|
||||
for i, value in enumerate(data):
|
||||
new_path = path + [i]
|
||||
new_value, new_encoded = self._encode(value, new_path)
|
||||
new_data.append(new_value)
|
||||
encoded.extend(new_encoded)
|
||||
return new_data, encoded
|
||||
|
||||
def _encode_dict(self, data, path):
|
||||
"""Encode any bytes in a dict, noting the index of what is encoded."""
|
||||
new_data = {}
|
||||
encoded = []
|
||||
for key, value in data.items():
|
||||
new_path = path + [key]
|
||||
new_value, new_encoded = self._encode(value, new_path)
|
||||
new_data[key] = new_value
|
||||
encoded.extend(new_encoded)
|
||||
return new_data, encoded
|
||||
|
||||
def _encode_bytes(self, data, path):
|
||||
"""Base64 encode a byte string."""
|
||||
return base64.b64encode(data).decode('utf-8'), [path]
|
||||
|
||||
|
||||
class TokenDecoder(object):
|
||||
"""Decodes token strings back into dictionaries.
|
||||
|
||||
This performs the inverse operation to the TokenEncoder, accepting
|
||||
opaque strings and decoding them into a useable form.
|
||||
"""
|
||||
|
||||
def decode(self, token):
|
||||
"""Decodes an opaque string to a dictionary.
|
||||
|
||||
:type token: str
|
||||
:param token: A token string given by the botocore pagination
|
||||
interface.
|
||||
|
||||
:rtype: dict
|
||||
:returns: A dictionary containing pagination information,
|
||||
particularly the service pagination token(s) but also other boto
|
||||
metadata.
|
||||
"""
|
||||
json_string = base64.b64decode(token.encode('utf-8')).decode('utf-8')
|
||||
decoded_token = json.loads(json_string)
|
||||
|
||||
# Remove the encoding metadata as it is read since it will no longer
|
||||
# be needed.
|
||||
encoded_keys = decoded_token.pop('boto_encoded_keys', None)
|
||||
if encoded_keys is None:
|
||||
return decoded_token
|
||||
else:
|
||||
return self._decode(decoded_token, encoded_keys)
|
||||
|
||||
def _decode(self, token, encoded_keys):
|
||||
"""Find each encoded value and decode it."""
|
||||
for key in encoded_keys:
|
||||
encoded = self._path_get(token, key)
|
||||
decoded = base64.b64decode(encoded.encode('utf-8'))
|
||||
self._path_set(token, key, decoded)
|
||||
return token
|
||||
|
||||
def _path_get(self, data, path):
|
||||
"""Return the nested data at the given path.
|
||||
|
||||
For instance:
|
||||
data = {'foo': ['bar', 'baz']}
|
||||
path = ['foo', 0]
|
||||
==> 'bar'
|
||||
"""
|
||||
# jmespath isn't used here because it would be difficult to actually
|
||||
# create the jmespath query when taking all of the unknowns of key
|
||||
# structure into account. Gross though this is, it is simple and not
|
||||
# very error prone.
|
||||
d = data
|
||||
for step in path:
|
||||
d = d[step]
|
||||
return d
|
||||
|
||||
def _path_set(self, data, path, value):
|
||||
"""Set the value of a key in the given data.
|
||||
|
||||
Example:
|
||||
data = {'foo': ['bar', 'baz']}
|
||||
path = ['foo', 1]
|
||||
value = 'bin'
|
||||
==> data = {'foo': ['bar', 'bin']}
|
||||
"""
|
||||
container = self._path_get(data, path[:-1])
|
||||
container[path[-1]] = value
|
||||
|
||||
|
||||
class PaginatorModel(object):
|
||||
def __init__(self, paginator_config):
|
||||
self._paginator_config = paginator_config['pagination']
|
||||
|
||||
def get_paginator(self, operation_name):
|
||||
try:
|
||||
single_paginator_config = self._paginator_config[operation_name]
|
||||
except KeyError:
|
||||
raise ValueError("Paginator for operation does not exist: %s"
|
||||
% operation_name)
|
||||
return single_paginator_config
|
||||
|
||||
|
||||
class PageIterator(object):
|
||||
def __init__(self, method, input_token, output_token, more_results,
|
||||
result_keys, non_aggregate_keys, limit_key, max_items,
|
||||
starting_token, page_size, op_kwargs):
|
||||
self._method = method
|
||||
self._input_token = input_token
|
||||
self._output_token = output_token
|
||||
self._more_results = more_results
|
||||
self._result_keys = result_keys
|
||||
self._max_items = max_items
|
||||
self._limit_key = limit_key
|
||||
self._starting_token = starting_token
|
||||
self._page_size = page_size
|
||||
self._op_kwargs = op_kwargs
|
||||
self._resume_token = None
|
||||
self._non_aggregate_key_exprs = non_aggregate_keys
|
||||
self._non_aggregate_part = {}
|
||||
self._token_encoder = TokenEncoder()
|
||||
self._token_decoder = TokenDecoder()
|
||||
|
||||
@property
|
||||
def result_keys(self):
|
||||
return self._result_keys
|
||||
|
||||
@property
|
||||
def resume_token(self):
|
||||
"""Token to specify to resume pagination."""
|
||||
return self._resume_token
|
||||
|
||||
@resume_token.setter
|
||||
def resume_token(self, value):
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("Bad starting token: %s" % value)
|
||||
|
||||
if 'boto_truncate_amount' in value:
|
||||
token_keys = sorted(self._input_token + ['boto_truncate_amount'])
|
||||
else:
|
||||
token_keys = sorted(self._input_token)
|
||||
dict_keys = sorted(value.keys())
|
||||
|
||||
if token_keys == dict_keys:
|
||||
self._resume_token = self._token_encoder.encode(value)
|
||||
else:
|
||||
raise ValueError("Bad starting token: %s" % value)
|
||||
|
||||
@property
|
||||
def non_aggregate_part(self):
|
||||
return self._non_aggregate_part
|
||||
|
||||
def __iter__(self):
|
||||
current_kwargs = self._op_kwargs
|
||||
previous_next_token = None
|
||||
next_token = dict((key, None) for key in self._input_token)
|
||||
if self._starting_token is not None:
|
||||
# If the starting token exists, populate the next_token with the
|
||||
# values inside it. This ensures that we have the service's
|
||||
# pagination token on hand if we need to truncate after the
|
||||
# first response.
|
||||
next_token = self._parse_starting_token()[0]
|
||||
# The number of items from result_key we've seen so far.
|
||||
total_items = 0
|
||||
first_request = True
|
||||
primary_result_key = self.result_keys[0]
|
||||
starting_truncation = 0
|
||||
self._inject_starting_params(current_kwargs)
|
||||
while True:
|
||||
response = self._make_request(current_kwargs)
|
||||
parsed = self._extract_parsed_response(response)
|
||||
if first_request:
|
||||
# The first request is handled differently. We could
|
||||
# possibly have a resume/starting token that tells us where
|
||||
# to index into the retrieved page.
|
||||
if self._starting_token is not None:
|
||||
starting_truncation = self._handle_first_request(
|
||||
parsed, primary_result_key, starting_truncation)
|
||||
first_request = False
|
||||
self._record_non_aggregate_key_values(parsed)
|
||||
else:
|
||||
# If this isn't the first request, we have already sliced into
|
||||
# the first request and had to make additional requests after.
|
||||
# We no longer need to add this to truncation.
|
||||
starting_truncation = 0
|
||||
current_response = primary_result_key.search(parsed)
|
||||
if current_response is None:
|
||||
current_response = []
|
||||
num_current_response = len(current_response)
|
||||
truncate_amount = 0
|
||||
if self._max_items is not None:
|
||||
truncate_amount = (total_items + num_current_response) \
|
||||
- self._max_items
|
||||
if truncate_amount > 0:
|
||||
self._truncate_response(parsed, primary_result_key,
|
||||
truncate_amount, starting_truncation,
|
||||
next_token)
|
||||
yield response
|
||||
break
|
||||
else:
|
||||
yield response
|
||||
total_items += num_current_response
|
||||
next_token = self._get_next_token(parsed)
|
||||
if all(t is None for t in next_token.values()):
|
||||
break
|
||||
if self._max_items is not None and \
|
||||
total_items == self._max_items:
|
||||
# We're on a page boundary so we can set the current
|
||||
# next token to be the resume token.
|
||||
self.resume_token = next_token
|
||||
break
|
||||
if previous_next_token is not None and \
|
||||
previous_next_token == next_token:
|
||||
message = ("The same next token was received "
|
||||
"twice: %s" % next_token)
|
||||
raise PaginationError(message=message)
|
||||
self._inject_token_into_kwargs(current_kwargs, next_token)
|
||||
previous_next_token = next_token
|
||||
|
||||
def search(self, expression):
|
||||
"""Applies a JMESPath expression to a paginator
|
||||
|
||||
Each page of results is searched using the provided JMESPath
|
||||
expression. If the result is not a list, it is yielded
|
||||
directly. If the result is a list, each element in the result
|
||||
is yielded individually (essentially implementing a flatmap in
|
||||
which the JMESPath search is the mapping function).
|
||||
|
||||
:type expression: str
|
||||
:param expression: JMESPath expression to apply to each page.
|
||||
|
||||
:return: Returns an iterator that yields the individual
|
||||
elements of applying a JMESPath expression to each page of
|
||||
results.
|
||||
"""
|
||||
compiled = jmespath.compile(expression)
|
||||
for page in self:
|
||||
results = compiled.search(page)
|
||||
if isinstance(results, list):
|
||||
for element in results:
|
||||
yield element
|
||||
else:
|
||||
# Yield result directly if it is not a list.
|
||||
yield results
|
||||
|
||||
def _make_request(self, current_kwargs):
|
||||
return self._method(**current_kwargs)
|
||||
|
||||
def _extract_parsed_response(self, response):
|
||||
return response
|
||||
|
||||
def _record_non_aggregate_key_values(self, response):
|
||||
non_aggregate_keys = {}
|
||||
for expression in self._non_aggregate_key_exprs:
|
||||
result = expression.search(response)
|
||||
set_value_from_jmespath(non_aggregate_keys,
|
||||
expression.expression,
|
||||
result)
|
||||
self._non_aggregate_part = non_aggregate_keys
|
||||
|
||||
def _inject_starting_params(self, op_kwargs):
|
||||
# If the user has specified a starting token we need to
|
||||
# inject that into the operation's kwargs.
|
||||
if self._starting_token is not None:
|
||||
# Don't need to do anything special if there is no starting
|
||||
# token specified.
|
||||
next_token = self._parse_starting_token()[0]
|
||||
self._inject_token_into_kwargs(op_kwargs, next_token)
|
||||
if self._page_size is not None:
|
||||
# Pass the page size as the parameter name for limiting
|
||||
# page size, also known as the limit_key.
|
||||
op_kwargs[self._limit_key] = self._page_size
|
||||
|
||||
def _inject_token_into_kwargs(self, op_kwargs, next_token):
|
||||
for name, token in next_token.items():
|
||||
if (token is not None) and (token != 'None'):
|
||||
op_kwargs[name] = token
|
||||
elif name in op_kwargs:
|
||||
del op_kwargs[name]
|
||||
|
||||
def _handle_first_request(self, parsed, primary_result_key,
|
||||
starting_truncation):
|
||||
# If the payload is an array or string, we need to slice into it
|
||||
# and only return the truncated amount.
|
||||
starting_truncation = self._parse_starting_token()[1]
|
||||
all_data = primary_result_key.search(parsed)
|
||||
if isinstance(all_data, (list, six.string_types)):
|
||||
data = all_data[starting_truncation:]
|
||||
else:
|
||||
data = None
|
||||
set_value_from_jmespath(
|
||||
parsed,
|
||||
primary_result_key.expression,
|
||||
data
|
||||
)
|
||||
# We also need to truncate any secondary result keys
|
||||
# because they were not truncated in the previous last
|
||||
# response.
|
||||
for token in self.result_keys:
|
||||
if token == primary_result_key:
|
||||
continue
|
||||
sample = token.search(parsed)
|
||||
if isinstance(sample, list):
|
||||
empty_value = []
|
||||
elif isinstance(sample, six.string_types):
|
||||
empty_value = ''
|
||||
elif isinstance(sample, (int, float)):
|
||||
empty_value = 0
|
||||
else:
|
||||
empty_value = None
|
||||
set_value_from_jmespath(parsed, token.expression, empty_value)
|
||||
return starting_truncation
|
||||
|
||||
def _truncate_response(self, parsed, primary_result_key, truncate_amount,
|
||||
starting_truncation, next_token):
|
||||
original = primary_result_key.search(parsed)
|
||||
if original is None:
|
||||
original = []
|
||||
amount_to_keep = len(original) - truncate_amount
|
||||
truncated = original[:amount_to_keep]
|
||||
set_value_from_jmespath(
|
||||
parsed,
|
||||
primary_result_key.expression,
|
||||
truncated
|
||||
)
|
||||
# The issue here is that even though we know how much we've truncated
|
||||
# we need to account for this globally including any starting
|
||||
# left truncation. For example:
|
||||
# Raw response: [0,1,2,3]
|
||||
# Starting index: 1
|
||||
# Max items: 1
|
||||
# Starting left truncation: [1, 2, 3]
|
||||
# End right truncation for max items: [1]
|
||||
# However, even though we only kept 1, this is post
|
||||
# left truncation so the next starting index should be 2, not 1
|
||||
# (left_truncation + amount_to_keep).
|
||||
next_token['boto_truncate_amount'] = \
|
||||
amount_to_keep + starting_truncation
|
||||
self.resume_token = next_token
|
||||
|
||||
def _get_next_token(self, parsed):
|
||||
if self._more_results is not None:
|
||||
if not self._more_results.search(parsed):
|
||||
return {}
|
||||
next_tokens = {}
|
||||
for output_token, input_key in \
|
||||
zip(self._output_token, self._input_token):
|
||||
next_token = output_token.search(parsed)
|
||||
# We do not want to include any empty strings as actual tokens.
|
||||
# Treat them as None.
|
||||
if next_token:
|
||||
next_tokens[input_key] = next_token
|
||||
else:
|
||||
next_tokens[input_key] = None
|
||||
return next_tokens
|
||||
|
||||
def result_key_iters(self):
|
||||
teed_results = tee(self, len(self.result_keys))
|
||||
return [ResultKeyIterator(i, result_key) for i, result_key
|
||||
in zip(teed_results, self.result_keys)]
|
||||
|
||||
def build_full_result(self):
|
||||
complete_result = {}
|
||||
for response in self:
|
||||
page = response
|
||||
# We want to try to catch operation object pagination
|
||||
# and format correctly for those. They come in the form
|
||||
# of a tuple of two elements: (http_response, parsed_responsed).
|
||||
# We want the parsed_response as that is what the page iterator
|
||||
# uses. We can remove it though once operation objects are removed.
|
||||
if isinstance(response, tuple) and len(response) == 2:
|
||||
page = response[1]
|
||||
# We're incrementally building the full response page
|
||||
# by page. For each page in the response we need to
|
||||
# inject the necessary components from the page
|
||||
# into the complete_result.
|
||||
for result_expression in self.result_keys:
|
||||
# In order to incrementally update a result key
|
||||
# we need to search the existing value from complete_result,
|
||||
# then we need to search the _current_ page for the
|
||||
# current result key value. Then we append the current
|
||||
# value onto the existing value, and re-set that value
|
||||
# as the new value.
|
||||
result_value = result_expression.search(page)
|
||||
if result_value is None:
|
||||
continue
|
||||
existing_value = result_expression.search(complete_result)
|
||||
if existing_value is None:
|
||||
# Set the initial result
|
||||
set_value_from_jmespath(
|
||||
complete_result, result_expression.expression,
|
||||
result_value)
|
||||
continue
|
||||
# Now both result_value and existing_value contain something
|
||||
if isinstance(result_value, list):
|
||||
existing_value.extend(result_value)
|
||||
elif isinstance(result_value, (int, float, six.string_types)):
|
||||
# Modify the existing result with the sum or concatenation
|
||||
set_value_from_jmespath(
|
||||
complete_result, result_expression.expression,
|
||||
existing_value + result_value)
|
||||
merge_dicts(complete_result, self.non_aggregate_part)
|
||||
if self.resume_token is not None:
|
||||
complete_result['NextToken'] = self.resume_token
|
||||
return complete_result
|
||||
|
||||
def _parse_starting_token(self):
|
||||
if self._starting_token is None:
|
||||
return None
|
||||
|
||||
# The starting token is a dict passed as a base64 encoded string.
|
||||
next_token = self._starting_token
|
||||
try:
|
||||
next_token = self._token_decoder.decode(next_token)
|
||||
index = 0
|
||||
if 'boto_truncate_amount' in next_token:
|
||||
index = next_token.get('boto_truncate_amount')
|
||||
del next_token['boto_truncate_amount']
|
||||
except (ValueError, TypeError):
|
||||
next_token, index = self._parse_starting_token_deprecated()
|
||||
return next_token, index
|
||||
|
||||
def _parse_starting_token_deprecated(self):
|
||||
"""
|
||||
This handles parsing of old style starting tokens, and attempts to
|
||||
coerce them into the new style.
|
||||
"""
|
||||
log.debug("Attempting to fall back to old starting token parser. For "
|
||||
"token: %s" % self._starting_token)
|
||||
if self._starting_token is None:
|
||||
return None
|
||||
|
||||
parts = self._starting_token.split('___')
|
||||
next_token = []
|
||||
index = 0
|
||||
if len(parts) == len(self._input_token) + 1:
|
||||
try:
|
||||
index = int(parts.pop())
|
||||
except ValueError:
|
||||
# This doesn't look like a valid old-style token, so we're
|
||||
# passing it along as an opaque service token.
|
||||
parts = [self._starting_token]
|
||||
|
||||
for part in parts:
|
||||
if part == 'None':
|
||||
next_token.append(None)
|
||||
else:
|
||||
next_token.append(part)
|
||||
return self._convert_deprecated_starting_token(next_token), index
|
||||
|
||||
def _convert_deprecated_starting_token(self, deprecated_token):
|
||||
"""
|
||||
This attempts to convert a deprecated starting token into the new
|
||||
style.
|
||||
"""
|
||||
len_deprecated_token = len(deprecated_token)
|
||||
len_input_token = len(self._input_token)
|
||||
if len_deprecated_token > len_input_token:
|
||||
raise ValueError("Bad starting token: %s" % self._starting_token)
|
||||
elif len_deprecated_token < len_input_token:
|
||||
log.debug("Old format starting token does not contain all input "
|
||||
"tokens. Setting the rest, in order, as None.")
|
||||
for i in range(len_input_token - len_deprecated_token):
|
||||
deprecated_token.append(None)
|
||||
return dict(zip(self._input_token, deprecated_token))
|
||||
|
||||
|
||||
class Paginator(object):
|
||||
PAGE_ITERATOR_CLS = PageIterator
|
||||
|
||||
def __init__(self, method, pagination_config, model):
|
||||
self._model = model
|
||||
self._method = method
|
||||
self._pagination_cfg = pagination_config
|
||||
self._output_token = self._get_output_tokens(self._pagination_cfg)
|
||||
self._input_token = self._get_input_tokens(self._pagination_cfg)
|
||||
self._more_results = self._get_more_results_token(self._pagination_cfg)
|
||||
self._non_aggregate_keys = self._get_non_aggregate_keys(
|
||||
self._pagination_cfg)
|
||||
self._result_keys = self._get_result_keys(self._pagination_cfg)
|
||||
self._limit_key = self._get_limit_key(self._pagination_cfg)
|
||||
|
||||
@property
|
||||
def result_keys(self):
|
||||
return self._result_keys
|
||||
|
||||
def _get_non_aggregate_keys(self, config):
|
||||
keys = []
|
||||
for key in config.get('non_aggregate_keys', []):
|
||||
keys.append(jmespath.compile(key))
|
||||
return keys
|
||||
|
||||
def _get_output_tokens(self, config):
|
||||
output = []
|
||||
output_token = config['output_token']
|
||||
if not isinstance(output_token, list):
|
||||
output_token = [output_token]
|
||||
for config in output_token:
|
||||
output.append(jmespath.compile(config))
|
||||
return output
|
||||
|
||||
def _get_input_tokens(self, config):
|
||||
input_token = self._pagination_cfg['input_token']
|
||||
if not isinstance(input_token, list):
|
||||
input_token = [input_token]
|
||||
return input_token
|
||||
|
||||
def _get_more_results_token(self, config):
|
||||
more_results = config.get('more_results')
|
||||
if more_results is not None:
|
||||
return jmespath.compile(more_results)
|
||||
|
||||
def _get_result_keys(self, config):
|
||||
result_key = config.get('result_key')
|
||||
if result_key is not None:
|
||||
if not isinstance(result_key, list):
|
||||
result_key = [result_key]
|
||||
result_key = [jmespath.compile(rk) for rk in result_key]
|
||||
return result_key
|
||||
|
||||
def _get_limit_key(self, config):
|
||||
return config.get('limit_key')
|
||||
|
||||
def paginate(self, **kwargs):
|
||||
"""Create paginator object for an operation.
|
||||
|
||||
This returns an iterable object. Iterating over
|
||||
this object will yield a single page of a response
|
||||
at a time.
|
||||
|
||||
"""
|
||||
page_params = self._extract_paging_params(kwargs)
|
||||
return self.PAGE_ITERATOR_CLS(
|
||||
self._method, self._input_token,
|
||||
self._output_token, self._more_results,
|
||||
self._result_keys, self._non_aggregate_keys,
|
||||
self._limit_key,
|
||||
page_params['MaxItems'],
|
||||
page_params['StartingToken'],
|
||||
page_params['PageSize'],
|
||||
kwargs)
|
||||
|
||||
def _extract_paging_params(self, kwargs):
|
||||
pagination_config = kwargs.pop('PaginationConfig', {})
|
||||
max_items = pagination_config.get('MaxItems', None)
|
||||
if max_items is not None:
|
||||
max_items = int(max_items)
|
||||
page_size = pagination_config.get('PageSize', None)
|
||||
if page_size is not None:
|
||||
if self._limit_key is None:
|
||||
raise PaginationError(
|
||||
message="PageSize parameter is not supported for the "
|
||||
"pagination interface for this operation.")
|
||||
input_members = self._model.input_shape.members
|
||||
limit_key_shape = input_members.get(self._limit_key)
|
||||
if limit_key_shape.type_name == 'string':
|
||||
if not isinstance(page_size, six.string_types):
|
||||
page_size = str(page_size)
|
||||
else:
|
||||
page_size = int(page_size)
|
||||
return {
|
||||
'MaxItems': max_items,
|
||||
'StartingToken': pagination_config.get('StartingToken', None),
|
||||
'PageSize': page_size,
|
||||
}
|
||||
|
||||
|
||||
class ResultKeyIterator(object):
|
||||
"""Iterates over the results of paginated responses.
|
||||
|
||||
Each iterator is associated with a single result key.
|
||||
Iterating over this object will give you each element in
|
||||
the result key list.
|
||||
|
||||
:param pages_iterator: An iterator that will give you
|
||||
pages of results (a ``PageIterator`` class).
|
||||
:param result_key: The JMESPath expression representing
|
||||
the result key.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, pages_iterator, result_key):
|
||||
self._pages_iterator = pages_iterator
|
||||
self.result_key = result_key
|
||||
|
||||
def __iter__(self):
|
||||
for page in self._pages_iterator:
|
||||
results = self.result_key.search(page)
|
||||
if results is None:
|
||||
results = []
|
||||
for result in results:
|
||||
yield result
|
||||
Reference in New Issue
Block a user