24024
af7c11d7f9
- 新增图像生成接口，支持试用、积分和自定义API Key模式 - 实现生成图片结果异步上传至MinIO存储，带重试机制 - 优化积分预扣除和异常退还逻辑，保障用户积分准确 - 添加获取生成历史记录接口，支持时间范围和分页 - 提供本地字典配置接口，支持模型、比例、提示模板和尺寸 - 实现图片批量上传接口，支持S3兼容对象存储 feat(admin): 增加管理员角色管理与权限分配接口 - 实现角色列表查询、角色创建、更新及删除功能 - 增加权限列表查询接口 - 实现用户角色分配接口，便于统一管理用户权限 - 增加系统字典增删查改接口，支持分类过滤和排序 - 权限控制全面覆盖管理接口，保证安全访问 feat(auth): 完善用户登录注册及权限相关接口与页面 - 实现手机号验证码发送及校验功能，保障注册安全 - 支持手机号注册、登录及退出接口，集成日志记录 - 增加修改密码功能，验证原密码后更新 - 提供动态导航菜单接口，基于权限展示不同菜单 - 实现管理界面路由及日志、角色、字典管理页面访问权限控制 - 添加系统日志查询接口，支持关键词和等级筛选 feat(app): 初始化Flask应用并配置蓝图与数据库 - 创建应用程序工厂，加载配置，初始化数据库和Redis客户端 - 注册认证、API及管理员蓝图，整合路由 - 根路由渲染主页模板 - 应用上下文中自动创建数据库表，保证运行环境准备完毕 feat(database): 提供数据库创建与迁移支持脚本 - 新增数据库创建脚本，支持自动检测是否已存在 - 添加数据库表初始化脚本，支持创建和删除所有表 - 实现RBAC权限初始化，包含基础权限和角色创建 - 新增字段手动修复脚本，添加用户API Key和积分字段 - 强制迁移脚本支持清理连接和修复表结构，初始化默认数据及角色分配 feat(config): 新增系统配置参数 - 配置数据库、Redis、Session和MinIO相关参数 - 添加AI接口地址及试用Key配置 - 集成阿里云短信服务配置及开发模式相关参数 feat(extensions): 初始化数据库、Redis和MinIO客户端 - 创建全局SQLAlchemy数据库实例和Redis客户端 - 配置基于boto3的MinIO兼容S3客户端 chore(logs): 添加示例系统日志文件 - 记录用户请求、验证码发送成功与失败的日志信息
517 lines
19 KiB
Python
517 lines
19 KiB
Python
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
# may not use this file except in compliance with the License. A copy of
|
|
# the License is located at
|
|
#
|
|
# https://aws.amazon.com/apache2.0/
|
|
#
|
|
# or in the "license" file accompanying this file. This file is
|
|
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
# ANY KIND, either express or implied. See the License for the specific
|
|
# language governing permissions and limitations under the License.
|
|
"""Abstractions over S3's upload/download operations.
|
|
|
|
This module provides high level abstractions for efficient
|
|
uploads/downloads. It handles several things for the user:
|
|
|
|
* Automatically switching to multipart transfers when
|
|
a file is over a specific size threshold
|
|
* Uploading/downloading a file in parallel
|
|
* Progress callbacks to monitor transfers
|
|
* Retries. While botocore handles retries for streaming uploads,
|
|
it is not possible for it to handle retries for streaming
|
|
downloads. This module handles retries for both cases so
|
|
you don't need to implement any retry logic yourself.
|
|
|
|
This module has a reasonable set of defaults. It also allows you
|
|
to configure many aspects of the transfer process including:
|
|
|
|
* Multipart threshold size
|
|
* Max parallel downloads
|
|
* Socket timeouts
|
|
* Retry amounts
|
|
|
|
There is no support for s3->s3 multipart copies at this
|
|
time.
|
|
|
|
|
|
.. _ref_s3transfer_usage:
|
|
|
|
Usage
|
|
=====
|
|
|
|
The simplest way to use this module is:
|
|
|
|
.. code-block:: python
|
|
|
|
client = boto3.client('s3', 'us-west-2')
|
|
transfer = S3Transfer(client)
|
|
# Upload /tmp/myfile to s3://bucket/key
|
|
transfer.upload_file('/tmp/myfile', 'bucket', 'key')
|
|
|
|
# Download s3://bucket/key to /tmp/myfile
|
|
transfer.download_file('bucket', 'key', '/tmp/myfile')
|
|
|
|
The ``upload_file`` and ``download_file`` methods also accept
|
|
``**kwargs``, which will be forwarded through to the corresponding
|
|
client operation. Here are a few examples using ``upload_file``::
|
|
|
|
# Making the object public
|
|
transfer.upload_file('/tmp/myfile', 'bucket', 'key',
|
|
extra_args={'ACL': 'public-read'})
|
|
|
|
# Setting metadata
|
|
transfer.upload_file('/tmp/myfile', 'bucket', 'key',
|
|
extra_args={'Metadata': {'a': 'b', 'c': 'd'}})
|
|
|
|
# Setting content type
|
|
transfer.upload_file('/tmp/myfile.json', 'bucket', 'key',
|
|
extra_args={'ContentType': "application/json"})
|
|
|
|
|
|
The ``S3Transfer`` class also supports progress callbacks so you can
|
|
provide transfer progress to users. Both the ``upload_file`` and
|
|
``download_file`` methods take an optional ``callback`` parameter.
|
|
Here's an example of how to print a simple progress percentage
|
|
to the user:
|
|
|
|
.. code-block:: python
|
|
|
|
class ProgressPercentage(object):
|
|
def __init__(self, filename):
|
|
self._filename = filename
|
|
self._size = float(os.path.getsize(filename))
|
|
self._seen_so_far = 0
|
|
self._lock = threading.Lock()
|
|
|
|
def __call__(self, bytes_amount):
|
|
# To simplify we'll assume this is hooked up
|
|
# to a single filename.
|
|
with self._lock:
|
|
self._seen_so_far += bytes_amount
|
|
percentage = (self._seen_so_far / self._size) * 100
|
|
sys.stdout.write(
|
|
"\r%s %s / %s (%.2f%%)" % (
|
|
self._filename, self._seen_so_far, self._size,
|
|
percentage))
|
|
sys.stdout.flush()
|
|
|
|
|
|
transfer = S3Transfer(boto3.client('s3', 'us-west-2'))
|
|
# Upload /tmp/myfile to s3://bucket/key and print upload progress.
|
|
transfer.upload_file('/tmp/myfile', 'bucket', 'key',
|
|
callback=ProgressPercentage('/tmp/myfile'))
|
|
|
|
|
|
|
|
You can also provide a TransferConfig object to the S3Transfer
|
|
object that gives you more fine grained control over the
|
|
transfer. For example:
|
|
|
|
.. code-block:: python
|
|
|
|
client = boto3.client('s3', 'us-west-2')
|
|
config = TransferConfig(
|
|
multipart_threshold=8 * 1024 * 1024,
|
|
max_concurrency=10,
|
|
num_download_attempts=10,
|
|
)
|
|
transfer = S3Transfer(client, config)
|
|
transfer.upload_file('/tmp/foo', 'bucket', 'key')
|
|
|
|
|
|
"""
|
|
|
|
import logging
|
|
import threading
|
|
from os import PathLike, fspath, getpid
|
|
|
|
from botocore.compat import HAS_CRT
|
|
from botocore.exceptions import ClientError, MissingDependencyException
|
|
from s3transfer.exceptions import (
|
|
RetriesExceededError as S3TransferRetriesExceededError,
|
|
)
|
|
from s3transfer.futures import NonThreadedExecutor
|
|
from s3transfer.manager import TransferConfig as S3TransferConfig
|
|
from s3transfer.manager import TransferManager
|
|
from s3transfer.subscribers import BaseSubscriber
|
|
from s3transfer.utils import OSUtils
|
|
|
|
import boto3.s3.constants as constants
|
|
from boto3.compat import TRANSFER_CONFIG_SUPPORTS_CRT
|
|
from boto3.exceptions import (
|
|
RetriesExceededError,
|
|
S3UploadFailedError,
|
|
)
|
|
|
|
if HAS_CRT:
|
|
import awscrt.s3
|
|
|
|
from boto3.crt import create_crt_transfer_manager
|
|
|
|
KB = 1024
|
|
MB = KB * KB
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def create_transfer_manager(client, config, osutil=None):
|
|
"""Creates a transfer manager based on configuration
|
|
|
|
:type client: boto3.client
|
|
:param client: The S3 client to use
|
|
|
|
:type config: boto3.s3.transfer.TransferConfig
|
|
:param config: The transfer config to use
|
|
|
|
:type osutil: s3transfer.utils.OSUtils
|
|
:param osutil: The os utility to use
|
|
|
|
:rtype: s3transfer.manager.TransferManager
|
|
:returns: A transfer manager based on parameters provided
|
|
"""
|
|
if _should_use_crt(config):
|
|
crt_transfer_manager = create_crt_transfer_manager(client, config)
|
|
if crt_transfer_manager is not None:
|
|
logger.debug(
|
|
"Using CRT client. pid: %s, thread: %s",
|
|
getpid(),
|
|
threading.get_ident(),
|
|
)
|
|
return crt_transfer_manager
|
|
|
|
# If we don't resolve something above, fallback to the default.
|
|
logger.debug(
|
|
"Using default client. pid: %s, thread: %s",
|
|
getpid(),
|
|
threading.get_ident(),
|
|
)
|
|
return _create_default_transfer_manager(client, config, osutil)
|
|
|
|
|
|
def _should_use_crt(config):
|
|
# This feature requires awscrt>=0.19.18
|
|
has_min_crt = HAS_CRT and has_minimum_crt_version((0, 19, 18))
|
|
is_optimized_instance = has_min_crt and awscrt.s3.is_optimized_for_system()
|
|
pref_transfer_client = config.preferred_transfer_client.lower()
|
|
|
|
if (
|
|
pref_transfer_client == constants.CRT_TRANSFER_CLIENT
|
|
and not has_min_crt
|
|
):
|
|
msg = (
|
|
"CRT transfer client is configured but is missing minimum CRT "
|
|
f"version. CRT installed: {HAS_CRT}"
|
|
)
|
|
if HAS_CRT:
|
|
msg += f", with version: {awscrt.__version__}"
|
|
raise MissingDependencyException(msg=msg)
|
|
|
|
if (
|
|
is_optimized_instance
|
|
and pref_transfer_client == constants.AUTO_RESOLVE_TRANSFER_CLIENT
|
|
) or pref_transfer_client == constants.CRT_TRANSFER_CLIENT:
|
|
logger.debug(
|
|
"Attempting to use CRTTransferManager. Config settings may be ignored."
|
|
)
|
|
return True
|
|
|
|
logger.debug(
|
|
"Opting out of CRT Transfer Manager. "
|
|
"Preferred client: %s, CRT available: %s, Instance Optimized: %s",
|
|
pref_transfer_client,
|
|
HAS_CRT,
|
|
is_optimized_instance,
|
|
)
|
|
return False
|
|
|
|
|
|
def has_minimum_crt_version(minimum_version):
|
|
"""Not intended for use outside boto3."""
|
|
if not HAS_CRT:
|
|
return False
|
|
|
|
crt_version_str = awscrt.__version__
|
|
try:
|
|
crt_version_ints = map(int, crt_version_str.split("."))
|
|
crt_version_tuple = tuple(crt_version_ints)
|
|
except (TypeError, ValueError):
|
|
return False
|
|
|
|
return crt_version_tuple >= minimum_version
|
|
|
|
|
|
def _create_default_transfer_manager(client, config, osutil):
|
|
"""Create the default TransferManager implementation for s3transfer."""
|
|
executor_cls = None
|
|
if not config.use_threads:
|
|
executor_cls = NonThreadedExecutor
|
|
return TransferManager(client, config, osutil, executor_cls)
|
|
|
|
|
|
class TransferConfig(S3TransferConfig):
|
|
ALIAS = {
|
|
'max_concurrency': 'max_request_concurrency',
|
|
'max_io_queue': 'max_io_queue_size',
|
|
}
|
|
DEFAULTS = {
|
|
'multipart_threshold': 8 * MB,
|
|
'max_concurrency': 10,
|
|
'max_request_concurrency': 10,
|
|
'multipart_chunksize': 8 * MB,
|
|
'num_download_attempts': 5,
|
|
'max_io_queue': 100,
|
|
'max_io_queue_size': 100,
|
|
'io_chunksize': 256 * KB,
|
|
'use_threads': True,
|
|
'max_bandwidth': None,
|
|
'preferred_transfer_client': constants.AUTO_RESOLVE_TRANSFER_CLIENT,
|
|
}
|
|
|
|
def __init__(
|
|
self,
|
|
multipart_threshold=None,
|
|
max_concurrency=None,
|
|
multipart_chunksize=None,
|
|
num_download_attempts=None,
|
|
max_io_queue=None,
|
|
io_chunksize=None,
|
|
use_threads=None,
|
|
max_bandwidth=None,
|
|
preferred_transfer_client=None,
|
|
):
|
|
"""Configuration object for managed S3 transfers
|
|
|
|
:param multipart_threshold: The transfer size threshold for which
|
|
multipart uploads, downloads, and copies will automatically be
|
|
triggered.
|
|
|
|
:param max_concurrency: The maximum number of threads that will be
|
|
making requests to perform a transfer. If ``use_threads`` is
|
|
set to ``False``, the value provided is ignored as the transfer
|
|
will only ever use the current thread.
|
|
|
|
:param multipart_chunksize: The partition size of each part for a
|
|
multipart transfer.
|
|
|
|
:param num_download_attempts: The number of download attempts that
|
|
will be retried upon errors with downloading an object in S3.
|
|
Note that these retries account for errors that occur when
|
|
streaming down the data from s3 (i.e. socket errors and read
|
|
timeouts that occur after receiving an OK response from s3).
|
|
Other retryable exceptions such as throttling errors and 5xx
|
|
errors are already retried by botocore (this default is 5). This
|
|
does not take into account the number of exceptions retried by
|
|
botocore. Note: This value is ignored when resolved transfer
|
|
manager type is CRTTransferManager.
|
|
|
|
:param max_io_queue: The maximum amount of read parts that can be
|
|
queued in memory to be written for a download. The size of each
|
|
of these read parts is at most the size of ``io_chunksize``.
|
|
Note: This value is ignored when resolved transfer manager type
|
|
is CRTTransferManager.
|
|
|
|
:param io_chunksize: The max size of each chunk in the io queue.
|
|
Currently, this is size used when ``read`` is called on the
|
|
downloaded stream as well. Note: This value is ignored when
|
|
resolved transfer manager type is CRTTransferManager.
|
|
|
|
:param use_threads: If True, threads will be used when performing
|
|
S3 transfers. If False, no threads will be used in
|
|
performing transfers; all logic will be run in the current thread.
|
|
Note: This value is ignored when resolved transfer manager type is
|
|
CRTTransferManager.
|
|
|
|
:param max_bandwidth: The maximum bandwidth that will be consumed
|
|
in uploading and downloading file content. The value is an integer
|
|
in terms of bytes per second. Note: This value is ignored when
|
|
resolved transfer manager type is CRTTransferManager.
|
|
|
|
:param preferred_transfer_client: String specifying preferred transfer
|
|
client for transfer operations.
|
|
|
|
Current supported settings are:
|
|
* auto (default) - Use the CRTTransferManager when calls
|
|
are made with supported environment and settings.
|
|
* classic - Only use the origin S3TransferManager with
|
|
requests. Disables possible CRT upgrade on requests.
|
|
* crt - Only use the CRTTransferManager with requests.
|
|
"""
|
|
init_args = {
|
|
'multipart_threshold': multipart_threshold,
|
|
'max_concurrency': max_concurrency,
|
|
'multipart_chunksize': multipart_chunksize,
|
|
'num_download_attempts': num_download_attempts,
|
|
'max_io_queue': max_io_queue,
|
|
'io_chunksize': io_chunksize,
|
|
'use_threads': use_threads,
|
|
'max_bandwidth': max_bandwidth,
|
|
'preferred_transfer_client': preferred_transfer_client,
|
|
}
|
|
resolved = self._resolve_init_args(init_args)
|
|
super().__init__(
|
|
multipart_threshold=resolved['multipart_threshold'],
|
|
max_request_concurrency=resolved['max_concurrency'],
|
|
multipart_chunksize=resolved['multipart_chunksize'],
|
|
num_download_attempts=resolved['num_download_attempts'],
|
|
max_io_queue_size=resolved['max_io_queue'],
|
|
io_chunksize=resolved['io_chunksize'],
|
|
max_bandwidth=resolved['max_bandwidth'],
|
|
)
|
|
# Some of the argument names are not the same as the inherited
|
|
# S3TransferConfig so we add aliases so you can still access the
|
|
# old version of the names.
|
|
for alias in self.ALIAS:
|
|
setattr(
|
|
self,
|
|
alias,
|
|
object.__getattribute__(self, self.ALIAS[alias]),
|
|
)
|
|
self.use_threads = resolved['use_threads']
|
|
self.preferred_transfer_client = resolved['preferred_transfer_client']
|
|
|
|
def __setattr__(self, name, value):
|
|
# If the alias name is used, make sure we set the name that it points
|
|
# to as that is what actually is used in governing the TransferManager.
|
|
if name in self.ALIAS:
|
|
super().__setattr__(self.ALIAS[name], value)
|
|
# Always set the value of the actual name provided.
|
|
super().__setattr__(name, value)
|
|
|
|
def __getattribute__(self, item):
|
|
value = object.__getattribute__(self, item)
|
|
if not TRANSFER_CONFIG_SUPPORTS_CRT:
|
|
return value
|
|
defaults = object.__getattribute__(self, 'DEFAULTS')
|
|
if item not in defaults:
|
|
return value
|
|
if value is self.UNSET_DEFAULT:
|
|
return defaults[item]
|
|
return value
|
|
|
|
def _resolve_init_args(self, init_args):
|
|
resolved = {}
|
|
for init_arg, val in init_args.items():
|
|
if val is not None:
|
|
resolved[init_arg] = val
|
|
elif TRANSFER_CONFIG_SUPPORTS_CRT:
|
|
resolved[init_arg] = self.UNSET_DEFAULT
|
|
else:
|
|
resolved[init_arg] = self.DEFAULTS[init_arg]
|
|
return resolved
|
|
|
|
|
|
class S3Transfer:
|
|
ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS
|
|
ALLOWED_UPLOAD_ARGS = TransferManager.ALLOWED_UPLOAD_ARGS
|
|
ALLOWED_COPY_ARGS = TransferManager.ALLOWED_COPY_ARGS
|
|
|
|
def __init__(self, client=None, config=None, osutil=None, manager=None):
|
|
if not client and not manager:
|
|
raise ValueError(
|
|
'Either a boto3.Client or s3transfer.manager.TransferManager '
|
|
'must be provided'
|
|
)
|
|
if manager and any([client, config, osutil]):
|
|
raise ValueError(
|
|
'Manager cannot be provided with client, config, '
|
|
'nor osutil. These parameters are mutually exclusive.'
|
|
)
|
|
if config is None:
|
|
config = TransferConfig()
|
|
if osutil is None:
|
|
osutil = OSUtils()
|
|
if manager:
|
|
self._manager = manager
|
|
else:
|
|
self._manager = create_transfer_manager(client, config, osutil)
|
|
|
|
def upload_file(
|
|
self, filename, bucket, key, callback=None, extra_args=None
|
|
):
|
|
"""Upload a file to an S3 object.
|
|
|
|
Variants have also been injected into S3 client, Bucket and Object.
|
|
You don't have to use S3Transfer.upload_file() directly.
|
|
|
|
.. seealso::
|
|
:py:meth:`S3.Client.upload_file`
|
|
:py:meth:`S3.Client.upload_fileobj`
|
|
"""
|
|
if isinstance(filename, PathLike):
|
|
filename = fspath(filename)
|
|
if not isinstance(filename, str):
|
|
raise ValueError('Filename must be a string or a path-like object')
|
|
|
|
subscribers = self._get_subscribers(callback)
|
|
future = self._manager.upload(
|
|
filename, bucket, key, extra_args, subscribers
|
|
)
|
|
try:
|
|
future.result()
|
|
# If a client error was raised, add the backwards compatibility layer
|
|
# that raises a S3UploadFailedError. These specific errors were only
|
|
# ever thrown for upload_parts but now can be thrown for any related
|
|
# client error.
|
|
except ClientError as e:
|
|
raise S3UploadFailedError(
|
|
f"Failed to upload {filename} to {bucket}/{key}: {e}"
|
|
)
|
|
|
|
def download_file(
|
|
self, bucket, key, filename, extra_args=None, callback=None
|
|
):
|
|
"""Download an S3 object to a file.
|
|
|
|
Variants have also been injected into S3 client, Bucket and Object.
|
|
You don't have to use S3Transfer.download_file() directly.
|
|
|
|
.. seealso::
|
|
:py:meth:`S3.Client.download_file`
|
|
:py:meth:`S3.Client.download_fileobj`
|
|
"""
|
|
if isinstance(filename, PathLike):
|
|
filename = fspath(filename)
|
|
if not isinstance(filename, str):
|
|
raise ValueError('Filename must be a string or a path-like object')
|
|
|
|
subscribers = self._get_subscribers(callback)
|
|
future = self._manager.download(
|
|
bucket, key, filename, extra_args, subscribers
|
|
)
|
|
try:
|
|
future.result()
|
|
# This is for backwards compatibility where when retries are
|
|
# exceeded we need to throw the same error from boto3 instead of
|
|
# s3transfer's built in RetriesExceededError as current users are
|
|
# catching the boto3 one instead of the s3transfer exception to do
|
|
# their own retries.
|
|
except S3TransferRetriesExceededError as e:
|
|
raise RetriesExceededError(e.last_exception)
|
|
|
|
def _get_subscribers(self, callback):
|
|
if not callback:
|
|
return None
|
|
return [ProgressCallbackInvoker(callback)]
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
self._manager.__exit__(*args)
|
|
|
|
|
|
class ProgressCallbackInvoker(BaseSubscriber):
|
|
"""A back-compat wrapper to invoke a provided callback via a subscriber
|
|
|
|
:param callback: A callable that takes a single positional argument for
|
|
how many bytes were transferred.
|
|
"""
|
|
|
|
def __init__(self, callback):
|
|
self._callback = callback
|
|
|
|
def on_progress(self, bytes_transferred, **kwargs):
|
|
self._callback(bytes_transferred)
|