24024
af7c11d7f9
- 新增图像生成接口，支持试用、积分和自定义API Key模式 - 实现生成图片结果异步上传至MinIO存储，带重试机制 - 优化积分预扣除和异常退还逻辑，保障用户积分准确 - 添加获取生成历史记录接口，支持时间范围和分页 - 提供本地字典配置接口，支持模型、比例、提示模板和尺寸 - 实现图片批量上传接口，支持S3兼容对象存储 feat(admin): 增加管理员角色管理与权限分配接口 - 实现角色列表查询、角色创建、更新及删除功能 - 增加权限列表查询接口 - 实现用户角色分配接口，便于统一管理用户权限 - 增加系统字典增删查改接口，支持分类过滤和排序 - 权限控制全面覆盖管理接口，保证安全访问 feat(auth): 完善用户登录注册及权限相关接口与页面 - 实现手机号验证码发送及校验功能，保障注册安全 - 支持手机号注册、登录及退出接口，集成日志记录 - 增加修改密码功能，验证原密码后更新 - 提供动态导航菜单接口，基于权限展示不同菜单 - 实现管理界面路由及日志、角色、字典管理页面访问权限控制 - 添加系统日志查询接口，支持关键词和等级筛选 feat(app): 初始化Flask应用并配置蓝图与数据库 - 创建应用程序工厂，加载配置，初始化数据库和Redis客户端 - 注册认证、API及管理员蓝图，整合路由 - 根路由渲染主页模板 - 应用上下文中自动创建数据库表，保证运行环境准备完毕 feat(database): 提供数据库创建与迁移支持脚本 - 新增数据库创建脚本，支持自动检测是否已存在 - 添加数据库表初始化脚本，支持创建和删除所有表 - 实现RBAC权限初始化，包含基础权限和角色创建 - 新增字段手动修复脚本，添加用户API Key和积分字段 - 强制迁移脚本支持清理连接和修复表结构，初始化默认数据及角色分配 feat(config): 新增系统配置参数 - 配置数据库、Redis、Session和MinIO相关参数 - 添加AI接口地址及试用Key配置 - 集成阿里云短信服务配置及开发模式相关参数 feat(extensions): 初始化数据库、Redis和MinIO客户端 - 创建全局SQLAlchemy数据库实例和Redis客户端 - 配置基于boto3的MinIO兼容S3客户端 chore(logs): 添加示例系统日志文件 - 记录用户请求、验证码发送成功与失败的日志信息
729 lines
27 KiB
Python
729 lines
27 KiB
Python
# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
# may not use this file except in compliance with the License. A copy of
|
|
# the License is located at
|
|
#
|
|
# http://aws.amazon.com/apache2.0/
|
|
#
|
|
# or in the "license" file accompanying this file. This file is
|
|
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
# ANY KIND, either express or implied. See the License for the specific
|
|
# language governing permissions and limitations under the License.
|
|
|
|
import base64
|
|
import json
|
|
import logging
|
|
from functools import partial
|
|
from itertools import tee
|
|
|
|
import jmespath
|
|
|
|
from botocore.context import with_current_context
|
|
from botocore.exceptions import PaginationError
|
|
from botocore.useragent import register_feature_id
|
|
from botocore.utils import merge_dicts, set_value_from_jmespath
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class TokenEncoder:
|
|
"""Encodes dictionaries into opaque strings.
|
|
|
|
This for the most part json dumps + base64 encoding, but also supports
|
|
having bytes in the dictionary in addition to the types that json can
|
|
handle by default.
|
|
|
|
This is intended for use in encoding pagination tokens, which in some
|
|
cases can be complex structures and / or contain bytes.
|
|
"""
|
|
|
|
def encode(self, token):
|
|
"""Encodes a dictionary to an opaque string.
|
|
|
|
:type token: dict
|
|
:param token: A dictionary containing pagination information,
|
|
particularly the service pagination token(s) but also other boto
|
|
metadata.
|
|
|
|
:rtype: str
|
|
:returns: An opaque string
|
|
"""
|
|
try:
|
|
# Try just using json dumps first to avoid having to traverse
|
|
# and encode the dict. In 99.9999% of cases this will work.
|
|
json_string = json.dumps(token)
|
|
except (TypeError, UnicodeDecodeError):
|
|
# If normal dumping failed, go through and base64 encode all bytes.
|
|
encoded_token, encoded_keys = self._encode(token, [])
|
|
|
|
# Save the list of all the encoded key paths. We can safely
|
|
# assume that no service will ever use this key.
|
|
encoded_token['boto_encoded_keys'] = encoded_keys
|
|
|
|
# Now that the bytes are all encoded, dump the json.
|
|
json_string = json.dumps(encoded_token)
|
|
|
|
# base64 encode the json string to produce an opaque token string.
|
|
return base64.b64encode(json_string.encode('utf-8')).decode('utf-8')
|
|
|
|
def _encode(self, data, path):
|
|
"""Encode bytes in given data, keeping track of the path traversed."""
|
|
if isinstance(data, dict):
|
|
return self._encode_dict(data, path)
|
|
elif isinstance(data, list):
|
|
return self._encode_list(data, path)
|
|
elif isinstance(data, bytes):
|
|
return self._encode_bytes(data, path)
|
|
else:
|
|
return data, []
|
|
|
|
def _encode_list(self, data, path):
|
|
"""Encode any bytes in a list, noting the index of what is encoded."""
|
|
new_data = []
|
|
encoded = []
|
|
for i, value in enumerate(data):
|
|
new_path = path + [i]
|
|
new_value, new_encoded = self._encode(value, new_path)
|
|
new_data.append(new_value)
|
|
encoded.extend(new_encoded)
|
|
return new_data, encoded
|
|
|
|
def _encode_dict(self, data, path):
|
|
"""Encode any bytes in a dict, noting the index of what is encoded."""
|
|
new_data = {}
|
|
encoded = []
|
|
for key, value in data.items():
|
|
new_path = path + [key]
|
|
new_value, new_encoded = self._encode(value, new_path)
|
|
new_data[key] = new_value
|
|
encoded.extend(new_encoded)
|
|
return new_data, encoded
|
|
|
|
def _encode_bytes(self, data, path):
|
|
"""Base64 encode a byte string."""
|
|
return base64.b64encode(data).decode('utf-8'), [path]
|
|
|
|
|
|
class TokenDecoder:
|
|
"""Decodes token strings back into dictionaries.
|
|
|
|
This performs the inverse operation to the TokenEncoder, accepting
|
|
opaque strings and decoding them into a useable form.
|
|
"""
|
|
|
|
def decode(self, token):
|
|
"""Decodes an opaque string to a dictionary.
|
|
|
|
:type token: str
|
|
:param token: A token string given by the botocore pagination
|
|
interface.
|
|
|
|
:rtype: dict
|
|
:returns: A dictionary containing pagination information,
|
|
particularly the service pagination token(s) but also other boto
|
|
metadata.
|
|
"""
|
|
json_string = base64.b64decode(token.encode('utf-8')).decode('utf-8')
|
|
decoded_token = json.loads(json_string)
|
|
|
|
# Remove the encoding metadata as it is read since it will no longer
|
|
# be needed.
|
|
encoded_keys = decoded_token.pop('boto_encoded_keys', None)
|
|
if encoded_keys is None:
|
|
return decoded_token
|
|
else:
|
|
return self._decode(decoded_token, encoded_keys)
|
|
|
|
def _decode(self, token, encoded_keys):
|
|
"""Find each encoded value and decode it."""
|
|
for key in encoded_keys:
|
|
encoded = self._path_get(token, key)
|
|
decoded = base64.b64decode(encoded.encode('utf-8'))
|
|
self._path_set(token, key, decoded)
|
|
return token
|
|
|
|
def _path_get(self, data, path):
|
|
"""Return the nested data at the given path.
|
|
|
|
For instance:
|
|
data = {'foo': ['bar', 'baz']}
|
|
path = ['foo', 0]
|
|
==> 'bar'
|
|
"""
|
|
# jmespath isn't used here because it would be difficult to actually
|
|
# create the jmespath query when taking all of the unknowns of key
|
|
# structure into account. Gross though this is, it is simple and not
|
|
# very error prone.
|
|
d = data
|
|
for step in path:
|
|
d = d[step]
|
|
return d
|
|
|
|
def _path_set(self, data, path, value):
|
|
"""Set the value of a key in the given data.
|
|
|
|
Example:
|
|
data = {'foo': ['bar', 'baz']}
|
|
path = ['foo', 1]
|
|
value = 'bin'
|
|
==> data = {'foo': ['bar', 'bin']}
|
|
"""
|
|
container = self._path_get(data, path[:-1])
|
|
container[path[-1]] = value
|
|
|
|
|
|
class PaginatorModel:
|
|
def __init__(self, paginator_config):
|
|
self._paginator_config = paginator_config['pagination']
|
|
|
|
def get_paginator(self, operation_name):
|
|
try:
|
|
single_paginator_config = self._paginator_config[operation_name]
|
|
except KeyError:
|
|
raise ValueError(
|
|
f"Paginator for operation does not exist: {operation_name}"
|
|
)
|
|
return single_paginator_config
|
|
|
|
|
|
class PageIterator:
|
|
"""An iterable object to paginate API results.
|
|
Please note it is NOT a python iterator.
|
|
Use ``iter`` to wrap this as a generator.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
method,
|
|
input_token,
|
|
output_token,
|
|
more_results,
|
|
result_keys,
|
|
non_aggregate_keys,
|
|
limit_key,
|
|
max_items,
|
|
starting_token,
|
|
page_size,
|
|
op_kwargs,
|
|
):
|
|
self._method = method
|
|
self._input_token = input_token
|
|
self._output_token = output_token
|
|
self._more_results = more_results
|
|
self._result_keys = result_keys
|
|
self._max_items = max_items
|
|
self._limit_key = limit_key
|
|
self._starting_token = starting_token
|
|
self._page_size = page_size
|
|
self._op_kwargs = op_kwargs
|
|
self._resume_token = None
|
|
self._non_aggregate_key_exprs = non_aggregate_keys
|
|
self._non_aggregate_part = {}
|
|
self._token_encoder = TokenEncoder()
|
|
self._token_decoder = TokenDecoder()
|
|
|
|
@property
|
|
def result_keys(self):
|
|
return self._result_keys
|
|
|
|
@property
|
|
def resume_token(self):
|
|
"""Token to specify to resume pagination."""
|
|
return self._resume_token
|
|
|
|
@resume_token.setter
|
|
def resume_token(self, value):
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"Bad starting token: {value}")
|
|
|
|
if 'boto_truncate_amount' in value:
|
|
token_keys = sorted(self._input_token + ['boto_truncate_amount'])
|
|
else:
|
|
token_keys = sorted(self._input_token)
|
|
dict_keys = sorted(value.keys())
|
|
|
|
if token_keys == dict_keys:
|
|
self._resume_token = self._token_encoder.encode(value)
|
|
else:
|
|
raise ValueError(f"Bad starting token: {value}")
|
|
|
|
@property
|
|
def non_aggregate_part(self):
|
|
return self._non_aggregate_part
|
|
|
|
def __iter__(self):
|
|
current_kwargs = self._op_kwargs
|
|
previous_next_token = None
|
|
next_token = {key: None for key in self._input_token}
|
|
if self._starting_token is not None:
|
|
# If the starting token exists, populate the next_token with the
|
|
# values inside it. This ensures that we have the service's
|
|
# pagination token on hand if we need to truncate after the
|
|
# first response.
|
|
next_token = self._parse_starting_token()[0]
|
|
# The number of items from result_key we've seen so far.
|
|
total_items = 0
|
|
first_request = True
|
|
primary_result_key = self.result_keys[0]
|
|
starting_truncation = 0
|
|
self._inject_starting_params(current_kwargs)
|
|
while True:
|
|
response = self._make_request(current_kwargs)
|
|
parsed = self._extract_parsed_response(response)
|
|
if first_request:
|
|
# The first request is handled differently. We could
|
|
# possibly have a resume/starting token that tells us where
|
|
# to index into the retrieved page.
|
|
if self._starting_token is not None:
|
|
starting_truncation = self._handle_first_request(
|
|
parsed, primary_result_key, starting_truncation
|
|
)
|
|
first_request = False
|
|
self._record_non_aggregate_key_values(parsed)
|
|
else:
|
|
# If this isn't the first request, we have already sliced into
|
|
# the first request and had to make additional requests after.
|
|
# We no longer need to add this to truncation.
|
|
starting_truncation = 0
|
|
current_response = primary_result_key.search(parsed)
|
|
if current_response is None:
|
|
current_response = []
|
|
num_current_response = len(current_response)
|
|
truncate_amount = 0
|
|
if self._max_items is not None:
|
|
truncate_amount = (
|
|
total_items + num_current_response - self._max_items
|
|
)
|
|
if truncate_amount > 0:
|
|
self._truncate_response(
|
|
parsed,
|
|
primary_result_key,
|
|
truncate_amount,
|
|
starting_truncation,
|
|
next_token,
|
|
)
|
|
yield response
|
|
break
|
|
else:
|
|
yield response
|
|
total_items += num_current_response
|
|
next_token = self._get_next_token(parsed)
|
|
if all(t is None for t in next_token.values()):
|
|
break
|
|
if (
|
|
self._max_items is not None
|
|
and total_items == self._max_items
|
|
):
|
|
# We're on a page boundary so we can set the current
|
|
# next token to be the resume token.
|
|
self.resume_token = next_token
|
|
break
|
|
if (
|
|
previous_next_token is not None
|
|
and previous_next_token == next_token
|
|
):
|
|
message = (
|
|
f"The same next token was received twice: {next_token}"
|
|
)
|
|
raise PaginationError(message=message)
|
|
self._inject_token_into_kwargs(current_kwargs, next_token)
|
|
previous_next_token = next_token
|
|
|
|
def search(self, expression):
|
|
"""Applies a JMESPath expression to a paginator
|
|
|
|
Each page of results is searched using the provided JMESPath
|
|
expression. If the result is not a list, it is yielded
|
|
directly. If the result is a list, each element in the result
|
|
is yielded individually (essentially implementing a flatmap in
|
|
which the JMESPath search is the mapping function).
|
|
|
|
:type expression: str
|
|
:param expression: JMESPath expression to apply to each page.
|
|
|
|
:return: Returns an iterator that yields the individual
|
|
elements of applying a JMESPath expression to each page of
|
|
results.
|
|
"""
|
|
compiled = jmespath.compile(expression)
|
|
for page in self:
|
|
results = compiled.search(page)
|
|
if isinstance(results, list):
|
|
yield from results
|
|
else:
|
|
# Yield result directly if it is not a list.
|
|
yield results
|
|
|
|
@with_current_context(partial(register_feature_id, 'PAGINATOR'))
|
|
def _make_request(self, current_kwargs):
|
|
return self._method(**current_kwargs)
|
|
|
|
def _extract_parsed_response(self, response):
|
|
return response
|
|
|
|
def _record_non_aggregate_key_values(self, response):
|
|
non_aggregate_keys = {}
|
|
for expression in self._non_aggregate_key_exprs:
|
|
result = expression.search(response)
|
|
set_value_from_jmespath(
|
|
non_aggregate_keys, expression.expression, result
|
|
)
|
|
self._non_aggregate_part = non_aggregate_keys
|
|
|
|
def _inject_starting_params(self, op_kwargs):
|
|
# If the user has specified a starting token we need to
|
|
# inject that into the operation's kwargs.
|
|
if self._starting_token is not None:
|
|
# Don't need to do anything special if there is no starting
|
|
# token specified.
|
|
next_token = self._parse_starting_token()[0]
|
|
self._inject_token_into_kwargs(op_kwargs, next_token)
|
|
if self._page_size is not None:
|
|
# Pass the page size as the parameter name for limiting
|
|
# page size, also known as the limit_key.
|
|
op_kwargs[self._limit_key] = self._page_size
|
|
|
|
def _inject_token_into_kwargs(self, op_kwargs, next_token):
|
|
for name, token in next_token.items():
|
|
if (token is not None) and (token != 'None'):
|
|
op_kwargs[name] = token
|
|
elif name in op_kwargs:
|
|
del op_kwargs[name]
|
|
|
|
def _handle_first_request(
|
|
self, parsed, primary_result_key, starting_truncation
|
|
):
|
|
# If the payload is an array or string, we need to slice into it
|
|
# and only return the truncated amount.
|
|
starting_truncation = self._parse_starting_token()[1]
|
|
all_data = primary_result_key.search(parsed)
|
|
if isinstance(all_data, (list, str)):
|
|
data = all_data[starting_truncation:]
|
|
else:
|
|
data = None
|
|
set_value_from_jmespath(parsed, primary_result_key.expression, data)
|
|
# We also need to truncate any secondary result keys
|
|
# because they were not truncated in the previous last
|
|
# response.
|
|
for token in self.result_keys:
|
|
if token == primary_result_key:
|
|
continue
|
|
sample = token.search(parsed)
|
|
if isinstance(sample, list):
|
|
empty_value = []
|
|
elif isinstance(sample, str):
|
|
empty_value = ''
|
|
elif isinstance(sample, (int, float)):
|
|
# Even though we may be resuming from a truncated page, we
|
|
# still start from the actual numeric secondary result. For
|
|
# DynamoDB's Count/ScannedCount, this will still show how many
|
|
# items the server evaluated, even if the client is truncating
|
|
# due to a StartingToken.
|
|
empty_value = sample
|
|
else:
|
|
empty_value = None
|
|
set_value_from_jmespath(parsed, token.expression, empty_value)
|
|
return starting_truncation
|
|
|
|
def _truncate_response(
|
|
self,
|
|
parsed,
|
|
primary_result_key,
|
|
truncate_amount,
|
|
starting_truncation,
|
|
next_token,
|
|
):
|
|
original = primary_result_key.search(parsed)
|
|
if original is None:
|
|
original = []
|
|
amount_to_keep = len(original) - truncate_amount
|
|
truncated = original[:amount_to_keep]
|
|
set_value_from_jmespath(
|
|
parsed, primary_result_key.expression, truncated
|
|
)
|
|
# The issue here is that even though we know how much we've truncated
|
|
# we need to account for this globally including any starting
|
|
# left truncation. For example:
|
|
# Raw response: [0,1,2,3]
|
|
# Starting index: 1
|
|
# Max items: 1
|
|
# Starting left truncation: [1, 2, 3]
|
|
# End right truncation for max items: [1]
|
|
# However, even though we only kept 1, this is post
|
|
# left truncation so the next starting index should be 2, not 1
|
|
# (left_truncation + amount_to_keep).
|
|
next_token['boto_truncate_amount'] = (
|
|
amount_to_keep + starting_truncation
|
|
)
|
|
self.resume_token = next_token
|
|
|
|
def _get_next_token(self, parsed):
|
|
if self._more_results is not None:
|
|
if not self._more_results.search(parsed):
|
|
return {}
|
|
next_tokens = {}
|
|
for output_token, input_key in zip(
|
|
self._output_token, self._input_token
|
|
):
|
|
next_token = output_token.search(parsed)
|
|
# We do not want to include any empty strings as actual tokens.
|
|
# Treat them as None.
|
|
if next_token:
|
|
next_tokens[input_key] = next_token
|
|
else:
|
|
next_tokens[input_key] = None
|
|
return next_tokens
|
|
|
|
def result_key_iters(self):
|
|
teed_results = tee(self, len(self.result_keys))
|
|
return [
|
|
ResultKeyIterator(i, result_key)
|
|
for i, result_key in zip(teed_results, self.result_keys)
|
|
]
|
|
|
|
def build_full_result(self):
|
|
complete_result = {}
|
|
for response in self:
|
|
page = response
|
|
# We want to try to catch operation object pagination
|
|
# and format correctly for those. They come in the form
|
|
# of a tuple of two elements: (http_response, parsed_responsed).
|
|
# We want the parsed_response as that is what the page iterator
|
|
# uses. We can remove it though once operation objects are removed.
|
|
if isinstance(response, tuple) and len(response) == 2:
|
|
page = response[1]
|
|
# We're incrementally building the full response page
|
|
# by page. For each page in the response we need to
|
|
# inject the necessary components from the page
|
|
# into the complete_result.
|
|
for result_expression in self.result_keys:
|
|
# In order to incrementally update a result key
|
|
# we need to search the existing value from complete_result,
|
|
# then we need to search the _current_ page for the
|
|
# current result key value. Then we append the current
|
|
# value onto the existing value, and re-set that value
|
|
# as the new value.
|
|
result_value = result_expression.search(page)
|
|
if result_value is None:
|
|
continue
|
|
existing_value = result_expression.search(complete_result)
|
|
if existing_value is None:
|
|
# Set the initial result
|
|
set_value_from_jmespath(
|
|
complete_result,
|
|
result_expression.expression,
|
|
result_value,
|
|
)
|
|
continue
|
|
# Now both result_value and existing_value contain something
|
|
if isinstance(result_value, list):
|
|
existing_value.extend(result_value)
|
|
elif isinstance(result_value, (int, float, str)):
|
|
# Modify the existing result with the sum or concatenation
|
|
set_value_from_jmespath(
|
|
complete_result,
|
|
result_expression.expression,
|
|
existing_value + result_value,
|
|
)
|
|
merge_dicts(complete_result, self.non_aggregate_part)
|
|
if self.resume_token is not None:
|
|
complete_result['NextToken'] = self.resume_token
|
|
return complete_result
|
|
|
|
def _parse_starting_token(self):
|
|
if self._starting_token is None:
|
|
return None
|
|
|
|
# The starting token is a dict passed as a base64 encoded string.
|
|
next_token = self._starting_token
|
|
try:
|
|
next_token = self._token_decoder.decode(next_token)
|
|
index = 0
|
|
if 'boto_truncate_amount' in next_token:
|
|
index = next_token.get('boto_truncate_amount')
|
|
del next_token['boto_truncate_amount']
|
|
except (ValueError, TypeError):
|
|
next_token, index = self._parse_starting_token_deprecated()
|
|
return next_token, index
|
|
|
|
def _parse_starting_token_deprecated(self):
|
|
"""
|
|
This handles parsing of old style starting tokens, and attempts to
|
|
coerce them into the new style.
|
|
"""
|
|
log.debug(
|
|
"Attempting to fall back to old starting token parser. For token: %s",
|
|
self._starting_token,
|
|
)
|
|
if self._starting_token is None:
|
|
return None
|
|
|
|
parts = self._starting_token.split('___')
|
|
next_token = []
|
|
index = 0
|
|
if len(parts) == len(self._input_token) + 1:
|
|
try:
|
|
index = int(parts.pop())
|
|
except ValueError:
|
|
# This doesn't look like a valid old-style token, so we're
|
|
# passing it along as an opaque service token.
|
|
parts = [self._starting_token]
|
|
|
|
for part in parts:
|
|
if part == 'None':
|
|
next_token.append(None)
|
|
else:
|
|
next_token.append(part)
|
|
return self._convert_deprecated_starting_token(next_token), index
|
|
|
|
def _convert_deprecated_starting_token(self, deprecated_token):
|
|
"""
|
|
This attempts to convert a deprecated starting token into the new
|
|
style.
|
|
"""
|
|
len_deprecated_token = len(deprecated_token)
|
|
len_input_token = len(self._input_token)
|
|
if len_deprecated_token > len_input_token:
|
|
raise ValueError(f"Bad starting token: {self._starting_token}")
|
|
elif len_deprecated_token < len_input_token:
|
|
log.debug(
|
|
"Old format starting token does not contain all input "
|
|
"tokens. Setting the rest, in order, as None."
|
|
)
|
|
for i in range(len_input_token - len_deprecated_token):
|
|
deprecated_token.append(None)
|
|
return dict(zip(self._input_token, deprecated_token))
|
|
|
|
|
|
class Paginator:
|
|
PAGE_ITERATOR_CLS = PageIterator
|
|
|
|
def __init__(self, method, pagination_config, model):
|
|
self._model = model
|
|
self._method = method
|
|
self._pagination_cfg = pagination_config
|
|
self._output_token = self._get_output_tokens(self._pagination_cfg)
|
|
self._input_token = self._get_input_tokens(self._pagination_cfg)
|
|
self._more_results = self._get_more_results_token(self._pagination_cfg)
|
|
self._non_aggregate_keys = self._get_non_aggregate_keys(
|
|
self._pagination_cfg
|
|
)
|
|
self._result_keys = self._get_result_keys(self._pagination_cfg)
|
|
self._limit_key = self._get_limit_key(self._pagination_cfg)
|
|
|
|
@property
|
|
def result_keys(self):
|
|
return self._result_keys
|
|
|
|
def _get_non_aggregate_keys(self, config):
|
|
keys = []
|
|
for key in config.get('non_aggregate_keys', []):
|
|
keys.append(jmespath.compile(key))
|
|
return keys
|
|
|
|
def _get_output_tokens(self, config):
|
|
output = []
|
|
output_token = config['output_token']
|
|
if not isinstance(output_token, list):
|
|
output_token = [output_token]
|
|
for config in output_token:
|
|
output.append(jmespath.compile(config))
|
|
return output
|
|
|
|
def _get_input_tokens(self, config):
|
|
input_token = self._pagination_cfg['input_token']
|
|
if not isinstance(input_token, list):
|
|
input_token = [input_token]
|
|
return input_token
|
|
|
|
def _get_more_results_token(self, config):
|
|
more_results = config.get('more_results')
|
|
if more_results is not None:
|
|
return jmespath.compile(more_results)
|
|
|
|
def _get_result_keys(self, config):
|
|
result_key = config.get('result_key')
|
|
if result_key is not None:
|
|
if not isinstance(result_key, list):
|
|
result_key = [result_key]
|
|
result_key = [jmespath.compile(rk) for rk in result_key]
|
|
return result_key
|
|
|
|
def _get_limit_key(self, config):
|
|
return config.get('limit_key')
|
|
|
|
def paginate(self, **kwargs):
|
|
"""Create paginator object for an operation.
|
|
|
|
This returns an iterable object. Iterating over
|
|
this object will yield a single page of a response
|
|
at a time.
|
|
|
|
"""
|
|
page_params = self._extract_paging_params(kwargs)
|
|
return self.PAGE_ITERATOR_CLS(
|
|
self._method,
|
|
self._input_token,
|
|
self._output_token,
|
|
self._more_results,
|
|
self._result_keys,
|
|
self._non_aggregate_keys,
|
|
self._limit_key,
|
|
page_params['MaxItems'],
|
|
page_params['StartingToken'],
|
|
page_params['PageSize'],
|
|
kwargs,
|
|
)
|
|
|
|
def _extract_paging_params(self, kwargs):
|
|
pagination_config = kwargs.pop('PaginationConfig', {})
|
|
max_items = pagination_config.get('MaxItems', None)
|
|
if max_items is not None:
|
|
max_items = int(max_items)
|
|
page_size = pagination_config.get('PageSize', None)
|
|
if page_size is not None:
|
|
if self._limit_key is None:
|
|
raise PaginationError(
|
|
message="PageSize parameter is not supported for the "
|
|
"pagination interface for this operation."
|
|
)
|
|
input_members = self._model.input_shape.members
|
|
limit_key_shape = input_members.get(self._limit_key)
|
|
if limit_key_shape.type_name == 'string':
|
|
if not isinstance(page_size, str):
|
|
page_size = str(page_size)
|
|
else:
|
|
page_size = int(page_size)
|
|
return {
|
|
'MaxItems': max_items,
|
|
'StartingToken': pagination_config.get('StartingToken', None),
|
|
'PageSize': page_size,
|
|
}
|
|
|
|
|
|
class ResultKeyIterator:
|
|
"""Iterates over the results of paginated responses.
|
|
|
|
Each iterator is associated with a single result key.
|
|
Iterating over this object will give you each element in
|
|
the result key list.
|
|
|
|
:param pages_iterator: An iterator that will give you
|
|
pages of results (a ``PageIterator`` class).
|
|
:param result_key: The JMESPath expression representing
|
|
the result key.
|
|
|
|
"""
|
|
|
|
def __init__(self, pages_iterator, result_key):
|
|
self._pages_iterator = pages_iterator
|
|
self.result_key = result_key
|
|
|
|
def __iter__(self):
|
|
for page in self._pages_iterator:
|
|
results = self.result_key.search(page)
|
|
if results is None:
|
|
results = []
|
|
yield from results
|