24024
af7c11d7f9
- 新增图像生成接口，支持试用、积分和自定义API Key模式 - 实现生成图片结果异步上传至MinIO存储，带重试机制 - 优化积分预扣除和异常退还逻辑，保障用户积分准确 - 添加获取生成历史记录接口，支持时间范围和分页 - 提供本地字典配置接口，支持模型、比例、提示模板和尺寸 - 实现图片批量上传接口，支持S3兼容对象存储 feat(admin): 增加管理员角色管理与权限分配接口 - 实现角色列表查询、角色创建、更新及删除功能 - 增加权限列表查询接口 - 实现用户角色分配接口，便于统一管理用户权限 - 增加系统字典增删查改接口，支持分类过滤和排序 - 权限控制全面覆盖管理接口，保证安全访问 feat(auth): 完善用户登录注册及权限相关接口与页面 - 实现手机号验证码发送及校验功能，保障注册安全 - 支持手机号注册、登录及退出接口，集成日志记录 - 增加修改密码功能，验证原密码后更新 - 提供动态导航菜单接口，基于权限展示不同菜单 - 实现管理界面路由及日志、角色、字典管理页面访问权限控制 - 添加系统日志查询接口，支持关键词和等级筛选 feat(app): 初始化Flask应用并配置蓝图与数据库 - 创建应用程序工厂，加载配置，初始化数据库和Redis客户端 - 注册认证、API及管理员蓝图，整合路由 - 根路由渲染主页模板 - 应用上下文中自动创建数据库表，保证运行环境准备完毕 feat(database): 提供数据库创建与迁移支持脚本 - 新增数据库创建脚本，支持自动检测是否已存在 - 添加数据库表初始化脚本，支持创建和删除所有表 - 实现RBAC权限初始化，包含基础权限和角色创建 - 新增字段手动修复脚本，添加用户API Key和积分字段 - 强制迁移脚本支持清理连接和修复表结构，初始化默认数据及角色分配 feat(config): 新增系统配置参数 - 配置数据库、Redis、Session和MinIO相关参数 - 添加AI接口地址及试用Key配置 - 集成阿里云短信服务配置及开发模式相关参数 feat(extensions): 初始化数据库、Redis和MinIO客户端 - 创建全局SQLAlchemy数据库实例和Redis客户端 - 配置基于boto3的MinIO兼容S3客户端 chore(logs): 添加示例系统日志文件 - 记录用户请求、验证码发送成功与失败的日志信息
382 lines
12 KiB
Python
382 lines
12 KiB
Python
from typing import List, Optional, Tuple, Union
|
|
|
|
from redis.commands.search.dialect import DEFAULT_DIALECT
|
|
|
|
|
|
class Query:
|
|
"""
|
|
Query is used to build complex queries that have more parameters than just
|
|
the query string. The query string is set in the constructor, and other
|
|
options have setter functions.
|
|
|
|
The setter functions return the query object so they can be chained.
|
|
i.e. `Query("foo").verbatim().filter(...)` etc.
|
|
"""
|
|
|
|
def __init__(self, query_string: str) -> None:
|
|
"""
|
|
Create a new query object.
|
|
The query string is set in the constructor, and other options have
|
|
setter functions.
|
|
"""
|
|
|
|
self._query_string: str = query_string
|
|
self._offset: int = 0
|
|
self._num: int = 10
|
|
self._no_content: bool = False
|
|
self._no_stopwords: bool = False
|
|
self._fields: Optional[List[str]] = None
|
|
self._verbatim: bool = False
|
|
self._with_payloads: bool = False
|
|
self._with_scores: bool = False
|
|
self._scorer: Optional[str] = None
|
|
self._filters: List = list()
|
|
self._ids: Optional[Tuple[str, ...]] = None
|
|
self._slop: int = -1
|
|
self._timeout: Optional[float] = None
|
|
self._in_order: bool = False
|
|
self._sortby: Optional[SortbyField] = None
|
|
self._return_fields: List = []
|
|
self._return_fields_decode_as: dict = {}
|
|
self._summarize_fields: List = []
|
|
self._highlight_fields: List = []
|
|
self._language: Optional[str] = None
|
|
self._expander: Optional[str] = None
|
|
self._dialect: int = DEFAULT_DIALECT
|
|
|
|
def query_string(self) -> str:
|
|
"""Return the query string of this query only."""
|
|
return self._query_string
|
|
|
|
def limit_ids(self, *ids) -> "Query":
|
|
"""Limit the results to a specific set of pre-known document
|
|
ids of any length."""
|
|
self._ids = ids
|
|
return self
|
|
|
|
def return_fields(self, *fields) -> "Query":
|
|
"""Add fields to return fields."""
|
|
for field in fields:
|
|
self.return_field(field)
|
|
return self
|
|
|
|
def return_field(
|
|
self,
|
|
field: str,
|
|
as_field: Optional[str] = None,
|
|
decode_field: Optional[bool] = True,
|
|
encoding: Optional[str] = "utf8",
|
|
) -> "Query":
|
|
"""
|
|
Add a field to the list of fields to return.
|
|
|
|
- **field**: The field to include in query results
|
|
- **as_field**: The alias for the field
|
|
- **decode_field**: Whether to decode the field from bytes to string
|
|
- **encoding**: The encoding to use when decoding the field
|
|
"""
|
|
self._return_fields.append(field)
|
|
self._return_fields_decode_as[field] = encoding if decode_field else None
|
|
if as_field is not None:
|
|
self._return_fields += ("AS", as_field)
|
|
return self
|
|
|
|
def _mk_field_list(self, fields: Optional[Union[List[str], str]]) -> List:
|
|
if not fields:
|
|
return []
|
|
return [fields] if isinstance(fields, str) else list(fields)
|
|
|
|
def summarize(
|
|
self,
|
|
fields: Optional[List] = None,
|
|
context_len: Optional[int] = None,
|
|
num_frags: Optional[int] = None,
|
|
sep: Optional[str] = None,
|
|
) -> "Query":
|
|
"""
|
|
Return an abridged format of the field, containing only the segments of
|
|
the field that contain the matching term(s).
|
|
|
|
If `fields` is specified, then only the mentioned fields are
|
|
summarized; otherwise, all results are summarized.
|
|
|
|
Server-side defaults are used for each option (except `fields`)
|
|
if not specified
|
|
|
|
- **fields** List of fields to summarize. All fields are summarized
|
|
if not specified
|
|
- **context_len** Amount of context to include with each fragment
|
|
- **num_frags** Number of fragments per document
|
|
- **sep** Separator string to separate fragments
|
|
"""
|
|
args = ["SUMMARIZE"]
|
|
fields = self._mk_field_list(fields)
|
|
if fields:
|
|
args += ["FIELDS", str(len(fields))] + fields
|
|
|
|
if context_len is not None:
|
|
args += ["LEN", str(context_len)]
|
|
if num_frags is not None:
|
|
args += ["FRAGS", str(num_frags)]
|
|
if sep is not None:
|
|
args += ["SEPARATOR", sep]
|
|
|
|
self._summarize_fields = args
|
|
return self
|
|
|
|
def highlight(
|
|
self, fields: Optional[List[str]] = None, tags: Optional[List[str]] = None
|
|
) -> "Query":
|
|
"""
|
|
Apply specified markup to matched term(s) within the returned field(s).
|
|
|
|
- **fields** If specified, then only those mentioned fields are
|
|
highlighted, otherwise all fields are highlighted
|
|
- **tags** A list of two strings to surround the match.
|
|
"""
|
|
args = ["HIGHLIGHT"]
|
|
fields = self._mk_field_list(fields)
|
|
if fields:
|
|
args += ["FIELDS", str(len(fields))] + fields
|
|
if tags:
|
|
args += ["TAGS"] + list(tags)
|
|
|
|
self._highlight_fields = args
|
|
return self
|
|
|
|
def language(self, language: str) -> "Query":
|
|
"""
|
|
Analyze the query as being in the specified language.
|
|
|
|
:param language: The language (e.g. `chinese` or `english`)
|
|
"""
|
|
self._language = language
|
|
return self
|
|
|
|
def slop(self, slop: int) -> "Query":
|
|
"""Allow a maximum of N intervening non-matched terms between
|
|
phrase terms (0 means exact phrase).
|
|
"""
|
|
self._slop = slop
|
|
return self
|
|
|
|
def timeout(self, timeout: float) -> "Query":
|
|
"""overrides the timeout parameter of the module"""
|
|
self._timeout = timeout
|
|
return self
|
|
|
|
def in_order(self) -> "Query":
|
|
"""
|
|
Match only documents where the query terms appear in
|
|
the same order in the document.
|
|
i.e., for the query "hello world", we do not match "world hello"
|
|
"""
|
|
self._in_order = True
|
|
return self
|
|
|
|
def scorer(self, scorer: str) -> "Query":
|
|
"""
|
|
Use a different scoring function to evaluate document relevance.
|
|
Default is `TFIDF`.
|
|
|
|
Since Redis 8.0 default was changed to BM25STD.
|
|
|
|
:param scorer: The scoring function to use
|
|
(e.g. `TFIDF.DOCNORM` or `BM25`)
|
|
"""
|
|
self._scorer = scorer
|
|
return self
|
|
|
|
def get_args(self) -> List[Union[str, int, float]]:
|
|
"""Format the redis arguments for this query and return them."""
|
|
args: List[Union[str, int, float]] = [self._query_string]
|
|
args += self._get_args_tags()
|
|
args += self._summarize_fields + self._highlight_fields
|
|
args += ["LIMIT", self._offset, self._num]
|
|
return args
|
|
|
|
def _get_args_tags(self) -> List[Union[str, int, float]]:
|
|
args: List[Union[str, int, float]] = []
|
|
if self._no_content:
|
|
args.append("NOCONTENT")
|
|
if self._fields:
|
|
args.append("INFIELDS")
|
|
args.append(len(self._fields))
|
|
args += self._fields
|
|
if self._verbatim:
|
|
args.append("VERBATIM")
|
|
if self._no_stopwords:
|
|
args.append("NOSTOPWORDS")
|
|
if self._filters:
|
|
for flt in self._filters:
|
|
if not isinstance(flt, Filter):
|
|
raise AttributeError("Did not receive a Filter object.")
|
|
args += flt.args
|
|
if self._with_payloads:
|
|
args.append("WITHPAYLOADS")
|
|
if self._scorer:
|
|
args += ["SCORER", self._scorer]
|
|
if self._with_scores:
|
|
args.append("WITHSCORES")
|
|
if self._ids:
|
|
args.append("INKEYS")
|
|
args.append(len(self._ids))
|
|
args += self._ids
|
|
if self._slop >= 0:
|
|
args += ["SLOP", self._slop]
|
|
if self._timeout is not None:
|
|
args += ["TIMEOUT", self._timeout]
|
|
if self._in_order:
|
|
args.append("INORDER")
|
|
if self._return_fields:
|
|
args.append("RETURN")
|
|
args.append(len(self._return_fields))
|
|
args += self._return_fields
|
|
if self._sortby:
|
|
if not isinstance(self._sortby, SortbyField):
|
|
raise AttributeError("Did not receive a SortByField.")
|
|
args.append("SORTBY")
|
|
args += self._sortby.args
|
|
if self._language:
|
|
args += ["LANGUAGE", self._language]
|
|
if self._expander:
|
|
args += ["EXPANDER", self._expander]
|
|
if self._dialect:
|
|
args += ["DIALECT", self._dialect]
|
|
|
|
return args
|
|
|
|
def paging(self, offset: int, num: int) -> "Query":
|
|
"""
|
|
Set the paging for the query (defaults to 0..10).
|
|
|
|
- **offset**: Paging offset for the results. Defaults to 0
|
|
- **num**: How many results do we want
|
|
"""
|
|
self._offset = offset
|
|
self._num = num
|
|
return self
|
|
|
|
def verbatim(self) -> "Query":
|
|
"""Set the query to be verbatim, i.e., use no query expansion
|
|
or stemming.
|
|
"""
|
|
self._verbatim = True
|
|
return self
|
|
|
|
def no_content(self) -> "Query":
|
|
"""Set the query to only return ids and not the document content."""
|
|
self._no_content = True
|
|
return self
|
|
|
|
def no_stopwords(self) -> "Query":
|
|
"""
|
|
Prevent the query from being filtered for stopwords.
|
|
Only useful in very big queries that you are certain contain
|
|
no stopwords.
|
|
"""
|
|
self._no_stopwords = True
|
|
return self
|
|
|
|
def with_payloads(self) -> "Query":
|
|
"""Ask the engine to return document payloads."""
|
|
self._with_payloads = True
|
|
return self
|
|
|
|
def with_scores(self) -> "Query":
|
|
"""Ask the engine to return document search scores."""
|
|
self._with_scores = True
|
|
return self
|
|
|
|
def limit_fields(self, *fields: str) -> "Query":
|
|
"""
|
|
Limit the search to specific TEXT fields only.
|
|
|
|
- **fields**: Each element should be a string, case sensitive field name
|
|
from the defined schema.
|
|
"""
|
|
self._fields = list(fields)
|
|
return self
|
|
|
|
def add_filter(self, flt: "Filter") -> "Query":
|
|
"""
|
|
Add a numeric or geo filter to the query.
|
|
**Currently, only one of each filter is supported by the engine**
|
|
|
|
- **flt**: A NumericFilter or GeoFilter object, used on a
|
|
corresponding field
|
|
"""
|
|
|
|
self._filters.append(flt)
|
|
return self
|
|
|
|
def sort_by(self, field: str, asc: bool = True) -> "Query":
|
|
"""
|
|
Add a sortby field to the query.
|
|
|
|
- **field** - the name of the field to sort by
|
|
- **asc** - when `True`, sorting will be done in ascending order
|
|
"""
|
|
self._sortby = SortbyField(field, asc)
|
|
return self
|
|
|
|
def expander(self, expander: str) -> "Query":
|
|
"""
|
|
Add an expander field to the query.
|
|
|
|
- **expander** - the name of the expander
|
|
"""
|
|
self._expander = expander
|
|
return self
|
|
|
|
def dialect(self, dialect: int) -> "Query":
|
|
"""
|
|
Add a dialect field to the query.
|
|
|
|
- **dialect** - dialect version to execute the query under
|
|
"""
|
|
self._dialect = dialect
|
|
return self
|
|
|
|
|
|
class Filter:
|
|
def __init__(self, keyword: str, field: str, *args: Union[str, float]) -> None:
|
|
self.args = [keyword, field] + list(args)
|
|
|
|
|
|
class NumericFilter(Filter):
|
|
INF = "+inf"
|
|
NEG_INF = "-inf"
|
|
|
|
def __init__(
|
|
self,
|
|
field: str,
|
|
minval: Union[int, str],
|
|
maxval: Union[int, str],
|
|
minExclusive: bool = False,
|
|
maxExclusive: bool = False,
|
|
) -> None:
|
|
args = [
|
|
minval if not minExclusive else f"({minval}",
|
|
maxval if not maxExclusive else f"({maxval}",
|
|
]
|
|
|
|
Filter.__init__(self, "FILTER", field, *args)
|
|
|
|
|
|
class GeoFilter(Filter):
|
|
METERS = "m"
|
|
KILOMETERS = "km"
|
|
FEET = "ft"
|
|
MILES = "mi"
|
|
|
|
def __init__(
|
|
self, field: str, lon: float, lat: float, radius: float, unit: str = KILOMETERS
|
|
) -> None:
|
|
Filter.__init__(self, "GEOFILTER", field, lon, lat, radius, unit)
|
|
|
|
|
|
class SortbyField:
|
|
def __init__(self, field: str, asc=True) -> None:
|
|
self.args = [field, "ASC" if asc else "DESC"]
|