24024
af7c11d7f9
- 新增图像生成接口，支持试用、积分和自定义API Key模式 - 实现生成图片结果异步上传至MinIO存储，带重试机制 - 优化积分预扣除和异常退还逻辑，保障用户积分准确 - 添加获取生成历史记录接口，支持时间范围和分页 - 提供本地字典配置接口，支持模型、比例、提示模板和尺寸 - 实现图片批量上传接口，支持S3兼容对象存储 feat(admin): 增加管理员角色管理与权限分配接口 - 实现角色列表查询、角色创建、更新及删除功能 - 增加权限列表查询接口 - 实现用户角色分配接口，便于统一管理用户权限 - 增加系统字典增删查改接口，支持分类过滤和排序 - 权限控制全面覆盖管理接口，保证安全访问 feat(auth): 完善用户登录注册及权限相关接口与页面 - 实现手机号验证码发送及校验功能，保障注册安全 - 支持手机号注册、登录及退出接口，集成日志记录 - 增加修改密码功能，验证原密码后更新 - 提供动态导航菜单接口，基于权限展示不同菜单 - 实现管理界面路由及日志、角色、字典管理页面访问权限控制 - 添加系统日志查询接口，支持关键词和等级筛选 feat(app): 初始化Flask应用并配置蓝图与数据库 - 创建应用程序工厂，加载配置，初始化数据库和Redis客户端 - 注册认证、API及管理员蓝图，整合路由 - 根路由渲染主页模板 - 应用上下文中自动创建数据库表，保证运行环境准备完毕 feat(database): 提供数据库创建与迁移支持脚本 - 新增数据库创建脚本，支持自动检测是否已存在 - 添加数据库表初始化脚本，支持创建和删除所有表 - 实现RBAC权限初始化，包含基础权限和角色创建 - 新增字段手动修复脚本，添加用户API Key和积分字段 - 强制迁移脚本支持清理连接和修复表结构，初始化默认数据及角色分配 feat(config): 新增系统配置参数 - 配置数据库、Redis、Session和MinIO相关参数 - 添加AI接口地址及试用Key配置 - 集成阿里云短信服务配置及开发模式相关参数 feat(extensions): 初始化数据库、Redis和MinIO客户端 - 创建全局SQLAlchemy数据库实例和Redis客户端 - 配置基于boto3的MinIO兼容S3客户端 chore(logs): 添加示例系统日志文件 - 记录用户请求、验证码发送成功与失败的日志信息
956 lines
30 KiB
Python
956 lines
30 KiB
Python
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
# may not use this file except in compliance with the License. A copy of
|
|
# the License is located at
|
|
#
|
|
# https://aws.amazon.com/apache2.0/
|
|
#
|
|
# or in the "license" file accompanying this file. This file is
|
|
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
# ANY KIND, either express or implied. See the License for the specific
|
|
# language governing permissions and limitations under the License.
|
|
import copy as python_copy
|
|
import logging
|
|
from functools import partial
|
|
|
|
from botocore.exceptions import ClientError
|
|
|
|
from boto3 import utils
|
|
from boto3.compat import is_append_mode
|
|
from boto3.s3.transfer import (
|
|
ProgressCallbackInvoker,
|
|
S3Transfer,
|
|
TransferConfig,
|
|
create_transfer_manager,
|
|
)
|
|
|
|
try:
|
|
from botocore.context import with_current_context
|
|
except ImportError:
|
|
from functools import wraps
|
|
|
|
def with_current_context(hook=None):
|
|
def decorator(func):
|
|
@wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
return func(*args, **kwargs)
|
|
|
|
return wrapper
|
|
|
|
return decorator
|
|
|
|
|
|
try:
|
|
from botocore.useragent import register_feature_id
|
|
except ImportError:
|
|
|
|
def register_feature_id(feature_id):
|
|
pass
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def inject_s3_transfer_methods(class_attributes, **kwargs):
|
|
utils.inject_attribute(class_attributes, 'upload_file', upload_file)
|
|
utils.inject_attribute(class_attributes, 'download_file', download_file)
|
|
utils.inject_attribute(class_attributes, 'copy', copy)
|
|
utils.inject_attribute(class_attributes, 'upload_fileobj', upload_fileobj)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_fileobj', download_fileobj
|
|
)
|
|
|
|
|
|
def inject_bucket_methods(class_attributes, **kwargs):
|
|
utils.inject_attribute(class_attributes, 'load', bucket_load)
|
|
utils.inject_attribute(class_attributes, 'upload_file', bucket_upload_file)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_file', bucket_download_file
|
|
)
|
|
utils.inject_attribute(class_attributes, 'copy', bucket_copy)
|
|
utils.inject_attribute(
|
|
class_attributes, 'upload_fileobj', bucket_upload_fileobj
|
|
)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_fileobj', bucket_download_fileobj
|
|
)
|
|
|
|
|
|
def inject_object_methods(class_attributes, **kwargs):
|
|
utils.inject_attribute(class_attributes, 'upload_file', object_upload_file)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_file', object_download_file
|
|
)
|
|
utils.inject_attribute(class_attributes, 'copy', object_copy)
|
|
utils.inject_attribute(
|
|
class_attributes, 'upload_fileobj', object_upload_fileobj
|
|
)
|
|
utils.inject_attribute(
|
|
class_attributes, 'download_fileobj', object_download_fileobj
|
|
)
|
|
|
|
|
|
def inject_object_summary_methods(class_attributes, **kwargs):
|
|
utils.inject_attribute(class_attributes, 'load', object_summary_load)
|
|
|
|
|
|
def bucket_load(self, *args, **kwargs):
|
|
"""
|
|
Calls s3.Client.list_buckets() to update the attributes of the Bucket
|
|
resource.
|
|
"""
|
|
# The docstring above is phrased this way to match what the autogenerated
|
|
# docs produce.
|
|
|
|
# We can't actually get the bucket's attributes from a HeadBucket,
|
|
# so we need to use a ListBuckets and search for our bucket.
|
|
# However, we may fail if we lack permissions to ListBuckets
|
|
# or the bucket is in another account. In which case, creation_date
|
|
# will be None.
|
|
self.meta.data = {}
|
|
try:
|
|
response = self.meta.client.list_buckets()
|
|
for bucket_data in response['Buckets']:
|
|
if bucket_data['Name'] == self.name:
|
|
self.meta.data = bucket_data
|
|
break
|
|
except ClientError as e:
|
|
if not e.response.get('Error', {}).get('Code') == 'AccessDenied':
|
|
raise
|
|
|
|
|
|
def object_summary_load(self, *args, **kwargs):
|
|
"""
|
|
Calls s3.Client.head_object to update the attributes of the ObjectSummary
|
|
resource.
|
|
"""
|
|
response = self.meta.client.head_object(
|
|
Bucket=self.bucket_name, Key=self.key
|
|
)
|
|
if 'ContentLength' in response:
|
|
response['Size'] = response.pop('ContentLength')
|
|
self.meta.data = response
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def upload_file(
|
|
self, Filename, Bucket, Key, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file to an S3 object.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.client('s3')
|
|
s3.upload_file('/tmp/hello.txt', 'amzn-s3-demo-bucket', 'hello.txt')
|
|
|
|
Similar behavior as S3Transfer's upload_file() method, except that
|
|
argument names are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to upload.
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to upload to.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to upload to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
with S3Transfer(self, Config) as transfer:
|
|
return transfer.upload_file(
|
|
filename=Filename,
|
|
bucket=Bucket,
|
|
key=Key,
|
|
extra_args=ExtraArgs,
|
|
callback=Callback,
|
|
)
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def download_file(
|
|
self, Bucket, Key, Filename, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an S3 object to a file.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.client('s3')
|
|
s3.download_file('amzn-s3-demo-bucket', 'hello.txt', '/tmp/hello.txt')
|
|
|
|
Similar behavior as S3Transfer's download_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to download from.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to download from.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to download to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
with S3Transfer(self, Config) as transfer:
|
|
return transfer.download_file(
|
|
bucket=Bucket,
|
|
key=Key,
|
|
filename=Filename,
|
|
extra_args=ExtraArgs,
|
|
callback=Callback,
|
|
)
|
|
|
|
|
|
def bucket_upload_file(
|
|
self, Filename, Key, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file to an S3 object.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
s3.Bucket('amzn-s3-demo-bucket').upload_file('/tmp/hello.txt', 'hello.txt')
|
|
|
|
Similar behavior as S3Transfer's upload_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to upload.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to upload to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
return self.meta.client.upload_file(
|
|
Filename=Filename,
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def bucket_download_file(
|
|
self, Key, Filename, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an S3 object to a file.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
s3.Bucket('amzn-s3-demo-bucket').download_file('hello.txt', '/tmp/hello.txt')
|
|
|
|
Similar behavior as S3Transfer's download_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to download from.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to download to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
return self.meta.client.download_file(
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
Filename=Filename,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_upload_file(
|
|
self, Filename, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file to an S3 object.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
s3.Object('amzn-s3-demo-bucket', 'hello.txt').upload_file('/tmp/hello.txt')
|
|
|
|
Similar behavior as S3Transfer's upload_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to upload.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
return self.meta.client.upload_file(
|
|
Filename=Filename,
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_download_file(
|
|
self, Filename, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an S3 object to a file.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
s3.Object('amzn-s3-demo-bucket', 'hello.txt').download_file('/tmp/hello.txt')
|
|
|
|
Similar behavior as S3Transfer's download_file() method,
|
|
except that parameters are capitalized. Detailed examples can be found at
|
|
:ref:`S3Transfer's Usage <ref_s3transfer_usage>`.
|
|
|
|
:type Filename: str
|
|
:param Filename: The path to the file to download to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
transfer.
|
|
"""
|
|
return self.meta.client.download_file(
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
Filename=Filename,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def copy(
|
|
self,
|
|
CopySource,
|
|
Bucket,
|
|
Key,
|
|
ExtraArgs=None,
|
|
Callback=None,
|
|
SourceClient=None,
|
|
Config=None,
|
|
):
|
|
"""Copy an object from one S3 location to another.
|
|
|
|
This is a managed transfer which will perform a multipart copy in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
copy_source = {
|
|
'Bucket': 'amzn-s3-demo-bucket1',
|
|
'Key': 'mykey'
|
|
}
|
|
s3.meta.client.copy(copy_source, 'amzn-s3-demo-bucket2', 'otherkey')
|
|
|
|
:type CopySource: dict
|
|
:param CopySource: The name of the source bucket, key name of the
|
|
source object, and optional version ID of the source object. The
|
|
dictionary format is:
|
|
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
|
|
that the ``VersionId`` key is optional and may be omitted.
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to copy to
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to copy to
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed copy arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the copy.
|
|
|
|
:type SourceClient: botocore or boto3 Client
|
|
:param SourceClient: The client to be used for operation that
|
|
may happen at the source object. For example, this client is
|
|
used for the head_object that determines the size of the copy.
|
|
If no client is provided, the current client is used as the client
|
|
for the source object.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
copy.
|
|
"""
|
|
subscribers = None
|
|
if Callback is not None:
|
|
subscribers = [ProgressCallbackInvoker(Callback)]
|
|
|
|
config = Config
|
|
if config is None:
|
|
config = TransferConfig()
|
|
|
|
# copy is not supported in the CRT
|
|
new_config = python_copy.copy(config)
|
|
new_config.preferred_transfer_client = "classic"
|
|
|
|
with create_transfer_manager(self, new_config) as manager:
|
|
future = manager.copy(
|
|
copy_source=CopySource,
|
|
bucket=Bucket,
|
|
key=Key,
|
|
extra_args=ExtraArgs,
|
|
subscribers=subscribers,
|
|
source_client=SourceClient,
|
|
)
|
|
return future.result()
|
|
|
|
|
|
def bucket_copy(
|
|
self,
|
|
CopySource,
|
|
Key,
|
|
ExtraArgs=None,
|
|
Callback=None,
|
|
SourceClient=None,
|
|
Config=None,
|
|
):
|
|
"""Copy an object from one S3 location to an object in this bucket.
|
|
|
|
This is a managed transfer which will perform a multipart copy in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
copy_source = {
|
|
'Bucket': 'amzn-s3-demo-bucket1',
|
|
'Key': 'mykey'
|
|
}
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket2')
|
|
bucket.copy(copy_source, 'otherkey')
|
|
|
|
:type CopySource: dict
|
|
:param CopySource: The name of the source bucket, key name of the
|
|
source object, and optional version ID of the source object. The
|
|
dictionary format is:
|
|
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
|
|
that the ``VersionId`` key is optional and may be omitted.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to copy to
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed copy arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the copy.
|
|
|
|
:type SourceClient: botocore or boto3 Client
|
|
:param SourceClient: The client to be used for operation that
|
|
may happen at the source object. For example, this client is
|
|
used for the head_object that determines the size of the copy.
|
|
If no client is provided, the current client is used as the client
|
|
for the source object.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
copy.
|
|
"""
|
|
return self.meta.client.copy(
|
|
CopySource=CopySource,
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
SourceClient=SourceClient,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_copy(
|
|
self,
|
|
CopySource,
|
|
ExtraArgs=None,
|
|
Callback=None,
|
|
SourceClient=None,
|
|
Config=None,
|
|
):
|
|
"""Copy an object from one S3 location to this object.
|
|
|
|
This is a managed transfer which will perform a multipart copy in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
copy_source = {
|
|
'Bucket': 'amzn-s3-demo-bucket1',
|
|
'Key': 'mykey'
|
|
}
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket2')
|
|
obj = bucket.Object('otherkey')
|
|
obj.copy(copy_source)
|
|
|
|
:type CopySource: dict
|
|
:param CopySource: The name of the source bucket, key name of the
|
|
source object, and optional version ID of the source object. The
|
|
dictionary format is:
|
|
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
|
|
that the ``VersionId`` key is optional and may be omitted.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed copy arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the copy.
|
|
|
|
:type SourceClient: botocore or boto3 Client
|
|
:param SourceClient: The client to be used for operation that
|
|
may happen at the source object. For example, this client is
|
|
used for the head_object that determines the size of the copy.
|
|
If no client is provided, the current client is used as the client
|
|
for the source object.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
copy.
|
|
"""
|
|
return self.meta.client.copy(
|
|
CopySource=CopySource,
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
SourceClient=SourceClient,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def upload_fileobj(
|
|
self, Fileobj, Bucket, Key, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file-like object to S3.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart upload in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.client('s3')
|
|
|
|
with open('filename', 'rb') as data:
|
|
s3.upload_fileobj(data, 'amzn-s3-demo-bucket', 'mykey')
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to upload. At a minimum, it must
|
|
implement the `read` method, and must return bytes.
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to upload to.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to upload to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
upload.
|
|
"""
|
|
if not hasattr(Fileobj, 'read'):
|
|
raise ValueError('Fileobj must implement read')
|
|
|
|
subscribers = None
|
|
if Callback is not None:
|
|
subscribers = [ProgressCallbackInvoker(Callback)]
|
|
|
|
config = Config
|
|
if config is None:
|
|
config = TransferConfig()
|
|
|
|
with create_transfer_manager(self, config) as manager:
|
|
future = manager.upload(
|
|
fileobj=Fileobj,
|
|
bucket=Bucket,
|
|
key=Key,
|
|
extra_args=ExtraArgs,
|
|
subscribers=subscribers,
|
|
)
|
|
return future.result()
|
|
|
|
|
|
def bucket_upload_fileobj(
|
|
self, Fileobj, Key, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file-like object to this bucket.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart upload in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket')
|
|
|
|
with open('filename', 'rb') as data:
|
|
bucket.upload_fileobj(data, 'mykey')
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to upload. At a minimum, it must
|
|
implement the `read` method, and must return bytes.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to upload to.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
upload.
|
|
"""
|
|
return self.meta.client.upload_fileobj(
|
|
Fileobj=Fileobj,
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_upload_fileobj(
|
|
self, Fileobj, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Upload a file-like object to this object.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart upload in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket')
|
|
obj = bucket.Object('mykey')
|
|
|
|
with open('filename', 'rb') as data:
|
|
obj.upload_fileobj(data)
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to upload. At a minimum, it must
|
|
implement the `read` method, and must return bytes.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed upload arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the upload.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
upload.
|
|
"""
|
|
return self.meta.client.upload_fileobj(
|
|
Fileobj=Fileobj,
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def disable_threading_if_append_mode(config, fileobj):
|
|
"""Set `TransferConfig.use_threads` to `False` if file-like
|
|
object is in append mode.
|
|
|
|
:type config: boto3.s3.transfer.TransferConfig
|
|
:param config: The transfer configuration to be used when performing the
|
|
download.
|
|
|
|
:type fileobj: A file-like object
|
|
:param fileobj: A file-like object to inspect for append mode.
|
|
"""
|
|
if is_append_mode(fileobj):
|
|
config.use_threads = False
|
|
logger.warning(
|
|
'A single thread will be used because the provided file object '
|
|
'is in append mode. Writes may always be appended to the end of '
|
|
'the file regardless of seek position, so a single thread must be '
|
|
'used to ensure sequential writes.'
|
|
)
|
|
|
|
|
|
@with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
|
|
def download_fileobj(
|
|
self, Bucket, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an object from S3 to a file-like object.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart download in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.client('s3')
|
|
|
|
with open('filename', 'wb') as data:
|
|
s3.download_fileobj('amzn-s3-demo-bucket', 'mykey', data)
|
|
|
|
:type Bucket: str
|
|
:param Bucket: The name of the bucket to download from.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to download from.
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to download into. At a minimum, it must
|
|
implement the `write` method and must accept bytes.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
download.
|
|
"""
|
|
if not hasattr(Fileobj, 'write'):
|
|
raise ValueError('Fileobj must implement write')
|
|
|
|
subscribers = None
|
|
if Callback is not None:
|
|
subscribers = [ProgressCallbackInvoker(Callback)]
|
|
|
|
config = Config
|
|
if config is None:
|
|
config = TransferConfig()
|
|
|
|
new_config = python_copy.copy(config)
|
|
disable_threading_if_append_mode(new_config, Fileobj)
|
|
|
|
with create_transfer_manager(self, new_config) as manager:
|
|
future = manager.download(
|
|
bucket=Bucket,
|
|
key=Key,
|
|
fileobj=Fileobj,
|
|
extra_args=ExtraArgs,
|
|
subscribers=subscribers,
|
|
)
|
|
return future.result()
|
|
|
|
|
|
def bucket_download_fileobj(
|
|
self, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download an object from this bucket to a file-like-object.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart download in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket')
|
|
|
|
with open('filename', 'wb') as data:
|
|
bucket.download_fileobj('mykey', data)
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to download into. At a minimum, it must
|
|
implement the `write` method and must accept bytes.
|
|
|
|
:type Key: str
|
|
:param Key: The name of the key to download from.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
download.
|
|
"""
|
|
return self.meta.client.download_fileobj(
|
|
Bucket=self.name,
|
|
Key=Key,
|
|
Fileobj=Fileobj,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|
|
|
|
|
|
def object_download_fileobj(
|
|
self, Fileobj, ExtraArgs=None, Callback=None, Config=None
|
|
):
|
|
"""Download this object from S3 to a file-like object.
|
|
|
|
The file-like object must be in binary mode.
|
|
|
|
This is a managed transfer which will perform a multipart download in
|
|
multiple threads if necessary.
|
|
|
|
Usage::
|
|
|
|
import boto3
|
|
s3 = boto3.resource('s3')
|
|
bucket = s3.Bucket('amzn-s3-demo-bucket')
|
|
obj = bucket.Object('mykey')
|
|
|
|
with open('filename', 'wb') as data:
|
|
obj.download_fileobj(data)
|
|
|
|
:type Fileobj: a file-like object
|
|
:param Fileobj: A file-like object to download into. At a minimum, it must
|
|
implement the `write` method and must accept bytes.
|
|
|
|
:type ExtraArgs: dict
|
|
:param ExtraArgs: Extra arguments that may be passed to the
|
|
client operation. For allowed download arguments see
|
|
:py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
|
|
|
|
:type Callback: function
|
|
:param Callback: A method which takes a number of bytes transferred to
|
|
be periodically called during the download.
|
|
|
|
:type Config: boto3.s3.transfer.TransferConfig
|
|
:param Config: The transfer configuration to be used when performing the
|
|
download.
|
|
"""
|
|
return self.meta.client.download_fileobj(
|
|
Bucket=self.bucket_name,
|
|
Key=self.key,
|
|
Fileobj=Fileobj,
|
|
ExtraArgs=ExtraArgs,
|
|
Callback=Callback,
|
|
Config=Config,
|
|
)
|